From 6fe3ce44e2815a2eac7ce4f1460f9a98c1cba497 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 09:54:47 +0200 Subject: [PATCH 01/51] feat(qwp): add schema-aware sender --- .../client/LineSenderSchemaException.java | 48 + .../io/questdb/client/cairo/ColumnType.java | 21 + .../cutlass/http/client/WebSocketClient.java | 70 + .../cutlass/qwp/client/QwpColumnWriter.java | 46 +- .../QwpSchemaCapabilityMismatchException.java | 29 + .../qwp/client/QwpWebSocketEncoder.java | 122 +- .../qwp/client/QwpWebSocketSender.java | 582 +- .../cutlass/qwp/client/WebSocketResponse.java | 161 +- .../client/sf/cursor/BackgroundDrainer.java | 33 +- .../client/sf/cursor/CursorSendEngine.java | 34 + .../sf/cursor/CursorWebSocketSendLoop.java | 207 +- .../sf/cursor/QwpSchemaCoordinator.java | 288 + .../sf/cursor/RecoveredFrameAnalysis.java | 10 + .../cutlass/qwp/protocol/QwpConstants.java | 2 + .../qwp/protocol/QwpSchemaBinding.java | 1367 ++++ .../protocol/QwpSchemaDoubleFormatter.java | 103 + .../qwp/protocol/QwpSchemaProtocol.java | 285 + .../qwp/protocol/QwpSchemaResponse.java | 101 + .../qwp/protocol/QwpSchemaRyuDouble.java | 635 ++ .../protocol/QwpSchemaTimestampFormatter.java | 187 + .../protocol/QwpSchemaTimestampParser.java | 599 ++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 182 +- .../io/questdb/client/std/DecimalParser.java | 51 +- .../WebSocketClientSchemaNegotiationTest.java | 228 + .../client/QwpSchemaFeedbackResponseTest.java | 259 + .../QwpSchemaSenderIntegrationTest.java | 2332 ++++++ .../qwp/client/QwpWebSocketEncoderTest.java | 147 + .../qwp/client/QwpWebSocketSenderTest.java | 105 +- .../qwp/client/WebSocketResponseTest.java | 7 +- .../CursorWebSocketSchemaCoordinatorTest.java | 408 + .../CursorWebSocketSchemaLifecycleTest.java | 454 ++ ...CursorWebSocketSendLoopErrorLatchTest.java | 251 - .../CursorWebSocketSendLoopJvmErrorTest.java | 167 +- .../sf/cursor/QwpSchemaReplayNetworkTest.java | 562 ++ .../sf/cursor/RecoveredFrameAnalysisTest.java | 103 + .../protocol/QwpSchemaBindingBinaryTest.java | 535 ++ .../protocol/QwpSchemaBindingBooleanTest.java | 372 + .../protocol/QwpSchemaBindingCharTest.java | 292 + .../QwpSchemaBindingFloatingDecimalTest.java | 408 + .../QwpSchemaBindingFloatingTest.java | 317 + .../QwpSchemaBindingFloatingTextTest.java | 466 ++ .../QwpSchemaBindingLongDecimalTest.java | 362 + .../QwpSchemaBindingLongTextTest.java | 400 + .../QwpSchemaBindingLongTimestampTest.java | 304 + .../QwpSchemaBindingNativeDecimalTest.java | 494 ++ .../QwpSchemaBindingStringDecimalTest.java | 342 + .../QwpSchemaBindingStringGeoHashTest.java | 365 + .../QwpSchemaBindingStringLong256Test.java | 309 + .../QwpSchemaBindingStringNumericTest.java | 408 + .../QwpSchemaBindingStringTimestampTest.java | 341 + .../qwp/protocol/QwpSchemaBindingTest.java | 1284 +++ .../protocol/QwpSchemaBindingTextTest.java | 477 ++ .../QwpSchemaBindingTimestampTextTest.java | 337 + .../QwpSchemaBindingUuidTextTest.java | 336 + .../qwp/protocol/QwpSchemaProtocolTest.java | 323 + .../qwp/websocket/TestWebSocketServer.java | 22 + .../client/test/std/Decimal128Test.java | 6 +- .../client/test/std/Decimal256Test.java | 26 +- .../client/test/std/Decimal64Test.java | 6 +- .../DecimalParserServerCompatibilityTest.java | 115 + .../client/cutlass/qwp/binary-input.tsv | 21 + .../client/cutlass/qwp/boolean-to-target.tsv | 19 + .../client/cutlass/qwp/char-conversions.tsv | 24 + .../cutlass/qwp/floating-to-decimal.tsv | 61 + .../cutlass/qwp/floating-to-numeric.tsv | 353 + .../client/cutlass/qwp/floating-to-text.tsv | 49 + .../client/cutlass/qwp/long-to-decimal.tsv | 40 + .../client/cutlass/qwp/long-to-numeric.tsv | 49 + .../client/cutlass/qwp/long-to-text.tsv | 19 + .../client/cutlass/qwp/long-to-timestamp.tsv | 13 + .../qwp/native-decimal-conversions.tsv | 61 + .../cutlass/qwp/native-decimal-to-text.tsv | 25 + .../qwp/string-geohash-conversions.tsv | 19 + .../qwp/string-long256-conversions.tsv | 24 + .../client/cutlass/qwp/string-to-boolean.tsv | 81 + .../client/cutlass/qwp/string-to-decimal.tsv | 56 + .../client/cutlass/qwp/string-to-numeric.tsv | 433 ++ .../qwp/string-to-timestamp-runtime.tsv | 8 + .../cutlass/qwp/string-to-timestamp.tsv | 111 + .../client/cutlass/qwp/text-to-text.tsv | 91 + .../client/cutlass/qwp/timestamp-inputs.tsv | 103 + .../client/cutlass/qwp/timestamp-to-text.tsv | 64 + .../client/cutlass/qwp/timestamp-units.tsv | 49 + .../cutlass/qwp/uuid-string-conformance.tsv | 21 + .../client/cutlass/qwp/uuid-to-text.tsv | 17 + design/schema-aware-sender-journal.md | 6928 +++++++++++++++++ ...ma-aware-sender-remaining-failures-plan.md | 693 ++ design/schema-aware-sender.md | 1703 ++++ 88 files changed, 29460 insertions(+), 508 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/LineSenderSchemaException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java delete mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java create mode 100644 core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv create mode 100644 design/schema-aware-sender-journal.md create mode 100644 design/schema-aware-sender-remaining-failures-plan.md create mode 100644 design/schema-aware-sender.md diff --git a/core/src/main/java/io/questdb/client/LineSenderSchemaException.java b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java new file mode 100644 index 000000000..4fd6e5121 --- /dev/null +++ b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java @@ -0,0 +1,48 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** A schema-bound row construction failure with a stable reason. */ +public class LineSenderSchemaException extends LineSenderException { + public enum Reason { + INVALID_VALUE, + SCHEMA_CHANGED, + SCHEMA_UNAVAILABLE, + ACCESS_DENIED, + UNSUPPORTED_FEATURE + } + + private final Reason reason; + + public LineSenderSchemaException(Reason reason, CharSequence message) { + super(message, false); + this.reason = reason; + } + + public Reason getReason() { + return reason; + } +} diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java index f87d6f322..1e4e4816b 100644 --- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java +++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java @@ -231,12 +231,24 @@ public static int getDecimalType(int precision, int scale) { return ((scale & 0xFF) << 18) | ((precision & 0xFF) << 8) | (DECIMAL8 + size); } + public static int getDecimalPrecision(int type) { + return (type >>> 8) & 0xFF; + } + + public static int getDecimalScale(int type) { + return (type >>> 18) & 0xFF; + } + public static int getGeoHashTypeWithBits(int bits) { assert bits > 0 && bits <= GEOLONG_MAX_BITS; // this logic relies on GeoHash type value to be clustered together return mkGeoHashType(bits, (short) (GEOBYTE + pow2SizeOfBits(bits))); } + public static int getGeoHashBits(int type) { + return (type >>> BYTE_BITS) & 0xff; + } + /** * Is an N-dimensional array type. */ @@ -244,6 +256,15 @@ public static boolean isArray(int columnType) { return ColumnType.tagOf(columnType) == ColumnType.ARRAY; } + public static boolean isDecimal(int columnType) { + short tag = tagOf(columnType); + return tag >= DECIMAL8 && tag <= DECIMAL256; + } + + public static boolean isGeoHash(int columnType) { + return (columnType & TYPE_FLAG_GEO_HASH) != 0; + } + public static boolean isNull(int columnType) { return columnType == NULL; } diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 1290fcab5..333f92f57 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -83,6 +83,8 @@ public abstract class WebSocketClient implements QuietCloseable { private static final String QWP_DURABLE_ACK_ENABLED_VALUE = "enabled"; private static final String QWP_DURABLE_ACK_HEADER_NAME = "X-QWP-Durable-Ack:"; private static final String QWP_MAX_BATCH_SIZE_HEADER_NAME = "X-QWP-Max-Batch-Size:"; + private static final String QWP_SCHEMA_ENABLED_VALUE = "enabled"; + private static final String QWP_SCHEMA_HEADER_NAME = "X-QWP-Schema:"; private static final String QWP_VERSION_HEADER_NAME = "X-QWP-Version:"; private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { try { @@ -138,6 +140,9 @@ public abstract class WebSocketClient implements QuietCloseable { private int qwpMaxVersion = 1; // Opt-in for STATUS_DURABLE_ACK frames; sent as X-QWP-Request-Durable-Ack: true private boolean qwpRequestDurableAck; + // Schema negotiation is deliberately opt-in at this low-level client layer. + // The public Sender does not activate it until schema-directed encoding is available. + private boolean qwpRequestSchema; // Receive buffer (native memory) private long recvBufPtr; private int recvBufSize; @@ -149,6 +154,7 @@ public abstract class WebSocketClient implements QuietCloseable { // path can rely on durable-ack-driven trim. Absence (after opting in via // setQwpRequestDurableAck) is the early-fail signal. private boolean serverDurableAckEnabled; + private boolean serverQwpSchemaEnabled; // Server's hard cap on ingest QWP message payload bytes, extracted from // X-QWP-Max-Batch-Size on the 101 upgrade response. 0 when the server did // not advertise the header (older builds), in which case the sender falls @@ -291,6 +297,7 @@ public void connect(CharSequence host, int port) { public void disconnect() { Misc.free(socket); upgraded = false; + serverQwpSchemaEnabled = false; host = null; port = 0; recvPos = 0; @@ -410,6 +417,11 @@ public boolean isServerDurableAckEnabled() { return serverDurableAckEnabled; } + /** Returns whether a requested schema extension was confirmed by this connection. */ + public boolean isQwpSchemaEnabled() { + return serverQwpSchemaEnabled; + } + /** * Receives and processes WebSocket frames. * @@ -469,6 +481,19 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** Sends a small caller-owned byte array as one binary message. */ + public void sendBinary(byte[] data, int timeout) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + for (byte b : data) { + sendBuffer.putByte(b); + } + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends two native-memory slices as one WebSocket binary frame. The slices * are copied directly into the WebSocket send buffer and masked there, so a @@ -595,6 +620,14 @@ public void setQwpRequestDurableAck(boolean enabled) { this.qwpRequestDurableAck = enabled; } + /** + * Requests schema control support on subsequent WebSocket upgrades. + * This operation is intentionally one-way; there is no low-level disable method. + */ + public void requestQwpSchema() { + this.qwpRequestSchema = true; + } + /** * Non-blocking attempt to receive a WebSocket frame. * Returns immediately if no complete frame is available. @@ -650,6 +683,7 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe upgradeRejectRole = null; upgradeRejectZone = null; upgradeStatusCode = 0; + serverQwpSchemaEnabled = false; // Generate random key byte[] keyBytes = new byte[16]; @@ -697,6 +731,9 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe if (qwpRequestDurableAck) { sendBuffer.putAscii("X-QWP-Request-Durable-Ack: true\r\n"); } + if (qwpRequestSchema) { + sendBuffer.putAscii("X-QWP-Request-Schema: true\r\n"); + } if (authorizationHeader != null) { sendBuffer.putAscii("Authorization: "); sendBuffer.putAscii(authorizationHeader); @@ -753,6 +790,28 @@ private static boolean excludesHeaderValue(String response, String headerName, S return true; } + private static String extractUniqueHeaderValue(String response, String headerName) { + String value = null; + int lineStart = response.indexOf("\r\n") + 2; // skip status line + while (lineStart >= 2 && lineStart < response.length()) { + int lineEnd = response.indexOf("\r\n", lineStart); + if (lineEnd < 0 || lineEnd == lineStart) { + break; + } + int colon = response.indexOf(':', lineStart); + if (colon > lineStart && colon < lineEnd + && colon - lineStart == headerName.length() - 1 + && response.regionMatches(true, lineStart, headerName, 0, headerName.length() - 1)) { + if (value != null) { + throw new HttpClientException("Duplicate X-QWP-Schema header"); + } + value = response.substring(colon + 1, lineEnd).trim(); + } + lineStart = lineEnd + 2; + } + return value; + } + /** * Extracts the zstd level the server applied from the echoed * {@code X-QWP-Content-Encoding} response header. Returns 0 when the @@ -1392,6 +1451,17 @@ private void validateUpgradeResponse(int headerEnd) { // missing trim signals. serverDurableAckEnabled = extractDurableAckEnabled(response); + // An absent confirmation means an older server. A present value is a + // protocol assertion, so a requested handshake must reject anything + // other than the single defined value instead of treating it as legacy. + if (qwpRequestSchema) { + String schemaValue = extractUniqueHeaderValue(response, QWP_SCHEMA_HEADER_NAME); + if (schemaValue != null && !schemaValue.equalsIgnoreCase(QWP_SCHEMA_ENABLED_VALUE)) { + throw new HttpClientException("Invalid X-QWP-Schema header value: ").put(schemaValue); + } + serverQwpSchemaEnabled = schemaValue != null; + } + // Extract X-QWP-Max-Batch-Size (optional). Older servers omit it; the // sender falls back to its locally configured byte budget in that case. serverMaxBatchSize = extractMaxBatchSize(response); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index f8cc669fb..11ec5ce13 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -53,11 +53,12 @@ private void encodeColumn( long stringDataSize, int symbolDictionarySize, boolean useGorilla, - boolean useGlobalSymbols + boolean useGlobalSymbols, + boolean forceNullBitmap ) { long dataAddr = col.getDataAddress(); - writeNullHeader(col, rowCount, rowCount - valueCount); + writeNullHeader(col, rowCount, rowCount - valueCount, forceNullBitmap); switch (col.getType()) { case TYPE_BOOLEAN: @@ -233,8 +234,13 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { } } - private void writeNullHeader(QwpTableBuffer.ColumnBuffer col, int rowCount, int nullCount) { - if (nullCount > 0) { + private void writeNullHeader( + QwpTableBuffer.ColumnBuffer col, + int rowCount, + int nullCount, + boolean forceNullBitmap + ) { + if (nullCount > 0 || forceNullBitmap) { buffer.putByte((byte) 1); col.ensureNullBitmapCapacity(rowCount); long nullAddr = col.getNullBitmapAddress(); @@ -281,6 +287,24 @@ private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int private void writeTableHeader(String tableName, int rowCount, QwpColumnDef[] columns) { buffer.putString(tableName); + writeTableShape(rowCount, columns); + } + + private void writeSchemaTableHeader(String tableName, int tableId, long metadataVersion, int rowCount, QwpColumnDef[] columns) { + buffer.putString(tableName); + if (tableId == -1 && metadataVersion == -1) { + buffer.putByte((byte) 0); + } else if (tableId >= 0 && metadataVersion >= 0) { + buffer.putByte((byte) 1); + buffer.putInt(tableId); + buffer.putLong(metadataVersion); + } else { + throw new IllegalArgumentException("schema identity must be both known or both unknown"); + } + writeTableShape(rowCount, columns); + } + + private void writeTableShape(int rowCount, QwpColumnDef[] columns) { buffer.putVarint(rowCount); buffer.putVarint(columns.length); for (QwpColumnDef col : columns) { @@ -319,6 +343,17 @@ void encodeTable(QwpTableBuffer tableBuffer, boolean useGlobalSymbols, boolean u encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useGlobalSymbols, useGorilla); } + void encodeSchemaTable(QwpTableBuffer tableBuffer, int tableId, long metadataVersion, boolean useGlobalSymbols, boolean useGorilla) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + writeSchemaTableHeader(tableBuffer.getTableName(), tableId, metadataVersion, tableBuffer.getRowCount(), columnDefs); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + encodeColumn(col, tableBuffer.getRowCount(), col.getValueCount(), col.getStringDataSize(), + col.getSymbolDictionarySize(), useGorilla, useGlobalSymbols, + col.getType() == TYPE_GEOHASH && col.usesNullBitmap()); + } + } + void encodeTable( QwpTableBuffer tableBuffer, int rowCount, @@ -344,7 +379,8 @@ void encodeTable( symbolDictionarySize = limitedSymbolDictionarySizes[i]; } - encodeColumn(col, rowCount, valueCount, stringDataSize, symbolDictionarySize, useGorilla, useGlobalSymbols); + encodeColumn(col, rowCount, valueCount, stringDataSize, symbolDictionarySize, + useGorilla, useGlobalSymbols, false); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java new file mode 100644 index 000000000..122beb47d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java @@ -0,0 +1,29 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; + +/** A connection cannot replay queued schema-framed data without schema support. */ +public final class QwpSchemaCapabilityMismatchException extends HttpClientException { + public QwpSchemaCapabilityMismatchException() { + super("WebSocket upgrade succeeded but server does not support schema-framed QWP backlog"); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index f38d9e93d..ad8cbfd11 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; @@ -56,6 +57,7 @@ public class QwpWebSocketEncoder implements QuietCloseable { // values when delta-of-delta overflows int32. private byte flags = FLAG_GORILLA; private int payloadStart; + private boolean schemaMessage; private byte version = VERSION; public QwpWebSocketEncoder() { @@ -67,23 +69,57 @@ public QwpWebSocketEncoder(int bufferSize) { } public void addTable(QwpTableBuffer tableBuffer) { + rejectBoundLegacyTable(tableBuffer); + if (schemaMessage) { + throw new IllegalStateException("legacy table cannot be added to a schema message"); + } columnWriter.encodeTable(tableBuffer, true, true); } + public void addSchemaTable(QwpTableBuffer tableBuffer, int tableId, long metadataVersion) { + validateBoundSchemaTable(tableBuffer, tableId, metadataVersion, true); + if (!schemaMessage) { + throw new IllegalStateException("schema table requires a schema message"); + } + validateIdentity(tableId, metadataVersion); + columnWriter.encodeSchemaTable(tableBuffer, tableId, metadataVersion, true, true); + } + public void beginMessage( int tableCount, GlobalSymbolDictionary globalDict, int confirmedMaxId, int batchMaxId + ) { + schemaMessage = false; + beginMessage0(tableCount, globalDict, confirmedMaxId, batchMaxId); + } + + public void beginSchemaMessage( + int tableCount, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId + ) { + if (tableCount <= 0) { + throw new IllegalArgumentException("schema message must contain at least one table"); + } + rejectControlFlag(); + schemaMessage = true; + beginMessage0(tableCount, globalDict, confirmedMaxId, batchMaxId); + } + + private void beginMessage0( + int tableCount, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId ) { buffer.reset(); deltaStart = confirmedMaxId + 1; deltaCount = Math.max(0, batchMaxId - confirmedMaxId); - byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); - byte origFlags = flags; - flags = headerFlags; - writeHeader(tableCount, 0); - flags = origFlags; + byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT | (schemaMessage ? FLAG_SCHEMA : 0)); + writeHeader(tableCount, 0, headerFlags); payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); @@ -163,6 +199,8 @@ public int copySplitMessage( } public int encode(QwpTableBuffer tableBuffer) { + rejectBoundLegacyTable(tableBuffer); + schemaMessage = false; buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); @@ -173,12 +211,38 @@ public int encode(QwpTableBuffer tableBuffer) { return buffer.getPosition(); } + public int encodeSchema(QwpTableBuffer tableBuffer, int tableId, long metadataVersion) { + validateIdentity(tableId, metadataVersion); + validateBoundSchemaTable(tableBuffer, tableId, metadataVersion, false); + if (tableBuffer.getSchemaBinding() != null && tableBuffer.getRowCount() == 0) { + return 0; + } + rejectControlFlag(); + schemaMessage = true; + buffer.reset(); + writeHeader(1, 0, (byte) (flags | FLAG_SCHEMA)); + int payloadStart = buffer.getPosition(); + columnWriter.setBuffer(buffer); + columnWriter.encodeSchemaTable(tableBuffer, tableId, metadataVersion, false, true); + buffer.patchInt(8, buffer.getPosition() - payloadStart); + return buffer.getPosition(); + } + + public int encodeSchema(QwpTableBuffer tableBuffer) { + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if (binding == null) { + throw new IllegalStateException("schema encoding requires an attached schema binding"); + } + return encodeSchema(tableBuffer, binding.getTableId(), binding.getMetadataVersion()); + } + public int encodeWithDeltaDict( QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, int confirmedMaxId, int batchMaxId ) { + rejectBoundLegacyTable(tableBuffer); beginMessage(1, globalDict, confirmedMaxId, batchMaxId); addTable(tableBuffer); return finishMessage(); @@ -190,6 +254,39 @@ public int finishMessage() { return buffer.getPosition(); } + private static void validateIdentity(int tableId, long metadataVersion) { + if (!((tableId == -1 && metadataVersion == -1) || (tableId >= 0 && metadataVersion >= 0))) { + throw new IllegalArgumentException("schema identity must be both known or both unknown"); + } + } + + private static void rejectBoundLegacyTable(QwpTableBuffer tableBuffer) { + if (tableBuffer.getSchemaBinding() != null) { + throw new IllegalStateException("schema-bound table cannot use legacy framing"); + } + } + + private static void validateBoundSchemaTable( + QwpTableBuffer tableBuffer, + int tableId, + long metadataVersion, + boolean additive + ) { + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if (binding == null) { + return; + } + if (binding.getTableId() != tableId || binding.getMetadataVersion() != metadataVersion) { + throw new IllegalArgumentException("explicit schema identity does not match the attached binding"); + } + if (tableBuffer.hasInProgressRow()) { + throw new IllegalStateException("cannot encode a schema-bound table with an incomplete row"); + } + if (additive && tableBuffer.getRowCount() == 0) { + throw new IllegalStateException("cannot add an empty schema-bound table"); + } + } + public QwpBufferWriter getBuffer() { return buffer; } @@ -234,16 +331,29 @@ public void setVersion(byte version) { } public void writeHeader(int tableCount, int payloadLength) { + writeHeader(tableCount, payloadLength, flags); + } + + private void writeHeader(int tableCount, int payloadLength, byte headerFlags) { + if (tableCount < 0 || tableCount > 0xffff) { + throw new IllegalArgumentException("QWP table count is outside unsigned-short range: " + tableCount); + } buffer.putByte((byte) 'Q'); buffer.putByte((byte) 'W'); buffer.putByte((byte) 'P'); buffer.putByte((byte) '1'); buffer.putByte(version); - buffer.putByte(flags); + buffer.putByte(headerFlags); buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + private void rejectControlFlag() { + if ((flags & io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol.FLAG_CONTROL) != 0) { + throw new IllegalStateException("schema data messages cannot use the control flag"); + } + } + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { if (splitDeltaCount == 0) { return 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 41cc0a8c9..91626d3a4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client; import io.questdb.client.ClientTlsConfiguration; +import io.questdb.client.LineSenderSchemaException; import io.questdb.client.Sender; import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; @@ -48,6 +49,9 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; @@ -197,6 +201,8 @@ public class QwpWebSocketSender implements Sender { // lockstep so index i names the same table in both, and in the same order as // splitFrameBodyBytes, which is what lets the split passes index them directly. private final ObjList flushTableBuffers = new ObjList<>(); + private final ObjList retiredTableBuffers = new ObjList<>(); + private final boolean[] schemaResolutionFresh = new boolean[1]; private final ObjList flushTableNames = new ObjList<>(); private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; @@ -385,6 +391,9 @@ public class QwpWebSocketSender implements Sender { private long reconnectMaxDurationMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS; private boolean requestDurableAck; + // One-way owner capability: shared by foreground and every background + // reconnect supplier created by this sender. + private volatile boolean schemaRequired; // Monotonic per-attempt counter snapshotted onto every connection event // fired from buildAndConnect. Counts every FOREGROUND endpoint try -- // successes and failures alike -- across this sender's lifetime. @@ -1014,12 +1023,21 @@ public void at(long timestamp, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.designatedTimestamp(timestamp, unit); + sendRow(); + return; + } if (unit == ChronoUnit.NANOS) { atNanos(timestamp); } else { long micros = toMicros(timestamp, unit); atMicros(micros); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, null, true); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1031,8 +1049,17 @@ public void at(Instant timestamp) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.designatedTimestamp(timestamp); + sendRow(); + return; + } long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; atMicros(micros); + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, null, true); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1044,6 +1071,7 @@ public void atNow() { checkNotClosed(); checkTableSelected(); try { + bindingForEffectiveWrite(); // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { @@ -1107,23 +1135,30 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { * Adds a BINARY column value to the current row. The bytes are written * verbatim with no encoding or transformation. A {@code null} array * reference is rejected so the NULL contract stays explicit (use the null - * bitmap instead). An empty array is accepted on the wire but QuestDB's - * BINARY storage uses the same NULL sentinel for zero-length and absent - * values, so an empty payload round-trips as NULL on read. + * bitmap instead). An empty array is accepted and remains a present, + * zero-length BINARY value, distinct from an omitted/null value. */ @Override public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { checkNotClosed(); checkTableSelected(); - if (value == null) { - throw new LineSenderException( - "BINARY value cannot be null; mark the row null via the null bitmap instead"); - } try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, value); + return this; + } + if (value == null) { + throw new LineSenderException( + "BINARY value cannot be null; mark the row null via the null bitmap instead"); + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BINARY, true); if (col != null) { col.addBinary(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1141,11 +1176,24 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice slice) { checkNotClosed(); checkTableSelected(); - if (slice == null) { - throw new LineSenderException( - "BINARY slice cannot be null; mark the row null via the null bitmap instead"); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, slice); + return this; + } + if (slice == null) { + throw new LineSenderException( + "BINARY slice cannot be null; mark the row null via the null bitmap instead"); + } + return binaryColumn(columnName, slice.ptr(), slice.size()); + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); + } catch (RuntimeException | Error e) { + rollbackRow(); + throw e; } - return binaryColumn(columnName, slice.ptr(), slice.size()); } /** @@ -1158,10 +1206,18 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, ptr, len); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BINARY, true); if (col != null) { col.addBinary(ptr, len); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1174,10 +1230,18 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.boolColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BOOLEAN, false); if (col != null) { col.addBoolean(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1201,6 +1265,11 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "BYTE"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BYTE, false); if (col != null) { col.addByte(value); @@ -1234,10 +1303,18 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.charColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_CHAR, false); if (col != null) { col.addShort((short) value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1470,6 +1547,10 @@ private void close0(boolean[] restoreInterrupt) { } } tableBuffers.clear(); + for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { + Misc.free(retiredTableBuffers.getQuick(i)); + } + retiredTableBuffers.clear(); } catch (Throwable t) { LOG.error("Error closing encoder or table buffers: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); @@ -1570,10 +1651,18 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL64, true); if (col != null) { col.addDecimal64(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1587,10 +1676,18 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL128, true); if (col != null) { col.addDecimal128(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1604,10 +1701,18 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1621,6 +1726,11 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { if (value == null || value.length() == 0) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "DECIMAL256"); + return this; + } currentDecimal256.ofString(value); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); if (col != null) { @@ -1639,6 +1749,11 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); @@ -1656,6 +1771,11 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); @@ -1673,6 +1793,11 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); @@ -1690,6 +1815,11 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(array); @@ -1706,10 +1836,18 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_DOUBLE, true); if (col != null) { col.addDouble(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1728,10 +1866,18 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.floatColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_FLOAT, true); if (col != null) { col.addFloat(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1869,11 +2015,16 @@ public boolean drain(long timeoutMillis) { public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int precisionBits) { checkNotClosed(); checkTableSelected(); - if (precisionBits < 1 || precisionBits > 60) { - throw new LineSenderException( - "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); - } try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "GEOHASH"); + return this; + } + if (precisionBits < 1 || precisionBits > 60) { + throw new LineSenderException( + "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_GEOHASH, true); if (col != null) { col.addGeoHash(maskGeoHashBits(bits, precisionBits), precisionBits); @@ -1903,6 +2054,16 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "GEOHASH"); + return this; + } + } catch (RuntimeException | Error e) { + rollbackRow(); + throw e; + } if (value == null) { throw new LineSenderException( "GEOHASH string cannot be null; mark the row null via the null bitmap instead"); @@ -2283,6 +2444,11 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "INT"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_INT, true); if (col != null) { col.addInt(value); @@ -2312,6 +2478,11 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "IPv4"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_IPv4, true); if (col != null) { col.addIPv4(address); @@ -2357,6 +2528,16 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre return this; } checkTableSelected(); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "IPv4"); + return this; + } + } catch (RuntimeException | Error e) { + rollbackRow(); + throw e; + } if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { throw new LineSenderException( "invalid IPv4 address: NULL sentinel inputs are rejected" @@ -2386,6 +2567,11 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "LONG256"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG256, true); if (col != null) { col.addLong256(l0, l1, l2, l3); @@ -2403,6 +2589,11 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); @@ -2420,6 +2611,11 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); @@ -2437,6 +2633,11 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); @@ -2454,6 +2655,11 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(array); @@ -2470,10 +2676,18 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.longColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG, true); if (col != null) { col.addLong(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2557,6 +2771,10 @@ public void reset() { buf.reset(); } } + for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { + Misc.free(retiredTableBuffers.getQuick(i)); + } + retiredTableBuffers.clear(); // Drop the batch's symbol watermark along with the rows that raised it. A // later flush encodes the delta section as // [sentMaxSymbolId+1 .. currentBatchMaxSymbolId], so a watermark left behind @@ -2774,6 +2992,11 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(columnName, "SHORT"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SHORT, false); if (col != null) { col.addShort(value); @@ -2890,10 +3113,18 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.stringColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_VARCHAR, true); if (col != null) { col.addString(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2906,10 +3137,18 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.symbol(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SYMBOL, true); if (col != null) { col.addSymbol(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2955,6 +3194,11 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.timestampColumn(columnName, value, unit); + return this; + } if (unit == ChronoUnit.NANOS) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_TIMESTAMP_NANOS, true); if (col != null) { @@ -2967,6 +3211,9 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C col.addLong(micros); } } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2979,11 +3226,19 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.timestampColumn(columnName, value); + return this; + } long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_TIMESTAMP, true); if (col != null) { col.addLong(micros); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -3003,10 +3258,18 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.uuidColumn(columnName, lo, hi); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_UUID, true); if (col != null) { col.addUuid(hi, lo); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -3372,6 +3635,10 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); + // Every Sender connection probes the schema extension. Absence on the + // first successful handshake selects legacy mode; once confirmation is + // observed, the existing engine/factory latch makes it mandatory. + newClient.requestQwpSchema(); newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); if (cancellation != null) { // Publish the client we are about to block on so a @@ -3499,6 +3766,22 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo // rethrow. close() is CAS-gated, so re-closing after the // durable-ack arm's own close is a no-op. try { + if (ctx.requiresSchema() && !newClient.isQwpSchemaEnabled()) { + newClient.close(); + hostTracker.recordRoleReject(idx, false, !background); + QwpSchemaCapabilityMismatchException schemaErr = new QwpSchemaCapabilityMismatchException(); + // Schema-backed bytes may never enter the durable-ack + // quarantine path, so schema evidence outranks every + // other capability gap in this sweep. + terminalUpgradeError = schemaErr; + lastError = schemaErr; + if (!background) { + dispatchConnectionEvent(SenderConnectionEvent.Kind.ENDPOINT_ATTEMPT_FAILED, + ep.host, ep.port, null, SenderConnectionEvent.NO_PORT, + attemptNumber, roundSeq, schemaErr); + } + continue; + } if (requestDurableAck && !newClient.isServerDurableAckEnabled()) { newClient.close(); hostTracker.recordRoleReject(idx, false, !background); @@ -3769,6 +4052,13 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError) private int collectNonEmptyTables(ObjList keys) { flushTableNames.clear(); flushTableBuffers.clear(); + for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { + QwpTableBuffer tableBuffer = retiredTableBuffers.getQuick(i); + if (tableBuffer != null && tableBuffer.getRowCount() > 0) { + flushTableNames.add(tableBuffer.getTableName()); + flushTableBuffers.add(tableBuffer); + } + } for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -4018,6 +4308,9 @@ private void ensureConnected() { connectionListener, connectionListenerInboxCapacity); } CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory(); + if (cursorEngine.requiresSchema()) { + reconnectFactory.requireSchema(); + } switch (initialConnectMode) { case SYNC: client = CursorWebSocketSendLoop.connectWithRetry( @@ -4152,6 +4445,120 @@ private void ensureConnected() { connected = true; } + /** + * Selects the wire contract before the first effective value of a row. + * Empty table selection remains local; an unreachable or failed handshake + * is never guessed to be a legacy server. + */ + private QwpSchemaBinding bindingForEffectiveWrite() { + QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); + if (currentTableBuffer.hasInProgressRow()) { + schemaResolutionFresh[0] = false; + return pinned; + } + final long deadlineNanos = System.nanoTime() + 1_000_000_000L; + ensureConnected(); + if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(deadlineNanos))) { + return null; + } + QwpSchemaResponse latest = cursorSendLoop.resolveSchema( + currentTableName, remainingSchemaMillis(deadlineNanos), schemaResolutionFresh); + if (pinned != null + && pinned.getTableId() == latest.getTableId() + && pinned.getMetadataVersion() == latest.getMetadataVersion()) { + return pinned; + } + return installSchemaBinding(latest, pinned); + } + + private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest, QwpSchemaBinding pinned) { + if (pinned == null && currentTableBuffer.getRowCount() == 0) { + return new QwpSchemaBinding(currentTableBuffer, latest); + } + QwpTableBuffer replacement = new QwpTableBuffer(currentTableName, this); + final QwpSchemaBinding replacementBinding; + try { + replacementBinding = new QwpSchemaBinding(replacement, latest); + } catch (RuntimeException | Error e) { + replacement.close(); + throw e; + } + QwpTableBuffer old = currentTableBuffer; + int retiredSlot = -1; + try { + if (old.getRowCount() > 0) { + retiredSlot = retiredTableBuffers.size(); + // Reserve list capacity before replacing the map entry. After the + // map mutation, the remaining assignments cannot allocate. + retiredTableBuffers.add(null); + } + tableBuffers.put(currentTableName, replacement); + } catch (RuntimeException | Error e) { + if (retiredSlot >= 0) { + retiredTableBuffers.remove(retiredSlot); + } + replacement.close(); + throw e; + } + currentTableBuffer = replacement; + if (retiredSlot >= 0) { + retiredTableBuffers.setQuick(retiredSlot, old); + } else { + old.close(); + } + currentTableBufferSnapshotBytes = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + return replacementBinding; + } + + private RuntimeException refreshAfterSchemaRejection( + LineSenderSchemaException rejection, + CharSequence columnName + ) { + return refreshAfterSchemaRejection(rejection, columnName, false); + } + + private RuntimeException refreshAfterSchemaRejection( + LineSenderSchemaException rejection, + CharSequence columnName, + boolean designatedTimestamp + ) { + if (schemaResolutionFresh[0] + || (rejection.getReason() != LineSenderSchemaException.Reason.INVALID_VALUE + && rejection.getReason() != LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) + || (!designatedTimestamp && (columnName == null + || !TableUtils.isValidColumnName(columnName, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)))) { + return rejection; + } + QwpSchemaBinding old = currentTableBuffer.getSchemaBinding(); + // An unbound failure has no pinned target to compare. Preserve its typed + // error instead of replacing it with a refresh failure or null dereference. + if (old == null) { + return rejection; + } + QwpSchemaResponse fresh = cursorSendLoop.refreshSchema(currentTableName, 1_000); + boolean identityChanged = old.getTableId() != fresh.getTableId() + || old.getMetadataVersion() != fresh.getMetadataVersion(); + boolean changed = old.getTableId() != fresh.getTableId() + || !old.hasSameRelevantTarget(fresh, columnName); + if (identityChanged) { + installSchemaBinding(fresh, old); + } + if (!changed) { + return rejection; + } + return new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_CHANGED, + "schema changed while rejecting row [table=" + currentTableName + + ", column=" + columnName + "]"); + } + + private static long remainingSchemaMillis(long deadlineNanos) { + long remaining = deadlineNanos - System.nanoTime(); + return remaining <= 0 ? 0 : 1L + (remaining - 1L) / 1_000_000L; + } + private void ensureNoInProgressRow() { if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { throw new LineSenderException( @@ -4192,6 +4599,10 @@ private void flushPendingRows(boolean deferCommit) { firstPendingRowTimeNanos = 0; return; } + if (tableCount > 0xffff) { + throw new LineSenderException("too many table blocks in one QWP message [count=") + .put(tableCount).put("]"); + } if (LOG.isDebugEnabled()) { LOG.debug("Flushing pending rows [count={}, tables={}, defer={}]", pendingRowCount, tableCount, deferCommit); @@ -4214,6 +4625,10 @@ private void flushPendingRows(boolean deferCommit) { // up the new cap. int cap = serverMaxBatchSize; int deltaBaseline = symbolDeltaBaseline(); + if (isMixedSchemaBatch(tableCount)) { + flushPendingRowsMixed(deferCommit, cap, deltaBaseline); + return; + } int combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); int messageSize = encoder.finishMessage(); @@ -4296,6 +4711,104 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { } } + /** + * Flushes the rare live legacy-to-schema upgrade batch. Each block is its + * own frame because a QWP frame has one wire family. The first pass sizes + * every frame before anything is published; the second pass re-encodes and + * publishes in collection order, with one commit boundary for the batch. + */ + private void flushPendingRowsMixed(boolean deferCommit, int cap, int deltaBaseline) { + final int blockCount = flushTableBuffers.size(); + int sizingBaseline = deltaBaseline; + boolean fits = mixedFramesFit(cap, sizingBaseline); + boolean dictionaryChunksAwaitCommit = false; + if (!fits && !deltaDictEnabled && currentBatchMaxSymbolId > deltaBaseline + && mixedFramesFit(cap, currentBatchMaxSymbolId)) { + publishDictionaryChunks(cap, deltaBaseline + 1, currentBatchMaxSymbolId); + dictionaryChunksAwaitCommit = true; + sizingBaseline = currentBatchMaxSymbolId; + } else if (!fits) { + throw new BatchTooLargeForCapException( + "single table block too large for server batch cap in mixed schema batch; " + + "the batch is retained for retry until a larger-cap node is reached"); + } + + int frameBaseline = sizingBaseline; + try { + for (int i = 0; i < blockCount; i++) { + boolean frameDefer = deferCommit || i + 1 < blockCount; + int messageSize = encodeSingleTableFrame(i, frameDefer, frameBaseline); + if (cap > 0 && messageSize > cap) { + throw new AssertionError("mixed frame exceeded preflight cap"); + } + persistNewSymbolsBeforePublish(); + QwpBufferWriter buffer = encoder.getBuffer(); + activeBuffer.ensureCapacity(messageSize); + activeBuffer.write(buffer.getBufferPtr(), messageSize); + activeBuffer.incrementRowCount(); + sealAndSwapBuffer(); + if (frameDefer) { + hasDeferredMessages = true; + } + advanceSentMaxSymbolId(); + if (deltaDictEnabled && currentBatchMaxSymbolId > frameBaseline) { + frameBaseline = currentBatchMaxSymbolId; + } + } + hasDeferredMessages = deferCommit; + if (!deferCommit) { + lastCommitBoundaryFsn = cursorEngine.publishedFsn(); + } + resetTableBuffersAfterFlush(); + } catch (Throwable t) { + if (dictionaryChunksAwaitCommit) { + commitOrphanedDictionaryChunks(t); + } + throw t; + } + } + + private boolean mixedFramesFit(int cap, int deltaBaseline) { + if (cap <= 0) { + return true; + } + int simBaseline = deltaBaseline; + for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { + int messageSize = encodeSingleTableFrame(i, true, simBaseline); + if (messageSize > cap) { + return false; + } + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; + } + } + return true; + } + + private int encodeSingleTableFrame(int index, boolean deferCommit, int deltaBaseline) { + encoder.setDeferCommit(deferCommit); + QwpTableBuffer tableBuffer = flushTableBuffers.getQuick(index); + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if (binding == null) { + encoder.beginMessage(1, globalSymbolDictionary, deltaBaseline, currentBatchMaxSymbolId); + encoder.addTable(tableBuffer); + } else { + encoder.beginSchemaMessage(1, globalSymbolDictionary, deltaBaseline, currentBatchMaxSymbolId); + encoder.addSchemaTable(tableBuffer, binding.getTableId(), binding.getMetadataVersion()); + } + return encoder.finishMessage(); + } + + private boolean isMixedSchemaBatch(int tableCount) { + boolean schema = flushTableBuffers.getQuick(0).getSchemaBinding() != null; + for (int i = 1; i < tableCount; i++) { + if ((flushTableBuffers.getQuick(i).getSchemaBinding() != null) != schema) { + return true; + } + } + return false; + } + /** * Encodes the staged batch as one combined frame at {@code deltaBaseline}: * begins the message, appends every non-empty table, and records each table's @@ -4309,8 +4822,14 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { */ private int encodeCombinedFrame(int tableCount, boolean deferCommit, int deltaBaseline) { encoder.setDeferCommit(deferCommit); - encoder.beginMessage(tableCount, globalSymbolDictionary, - deltaBaseline, currentBatchMaxSymbolId); + boolean schemaMessage = flushTableBuffers.getQuick(0).getSchemaBinding() != null; + if (schemaMessage) { + encoder.beginSchemaMessage(tableCount, globalSymbolDictionary, + deltaBaseline, currentBatchMaxSymbolId); + } else { + encoder.beginMessage(tableCount, globalSymbolDictionary, + deltaBaseline, currentBatchMaxSymbolId); + } splitFrameBodyBytes.clear(); int combinedBodyStart = encoder.getBuffer().getPosition(); int bodyStart = combinedBodyStart; @@ -4322,7 +4841,15 @@ private int encodeCombinedFrame(int tableCount, boolean deferCommit, int deltaBa flushTableNames.getQuick(i), tableBuffer.getRowCount(), currentBatchMaxSymbolId); } - encoder.addTable(tableBuffer); + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if ((binding != null) != schemaMessage) { + throw new IllegalStateException("cannot mix legacy and schema table blocks in one frame"); + } + if (binding != null) { + encoder.addSchemaTable(tableBuffer, binding.getTableId(), binding.getMetadataVersion()); + } else { + encoder.addTable(tableBuffer); + } int bodyEnd = encoder.getBuffer().getPosition(); splitFrameBodyBytes.add(bodyEnd - bodyStart); bodyStart = bodyEnd; @@ -4533,6 +5060,10 @@ private void resetTableBuffersAfterFlush() { // Drop the references; the next flush re-collects. flushTableNames.clear(); flushTableBuffers.clear(); + for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { + Misc.free(retiredTableBuffers.getQuick(i)); + } + retiredTableBuffers.clear(); currentBatchMaxSymbolId = -1; pendingBytes = 0; currentTableBufferSnapshotBytes = 0; @@ -5285,6 +5816,12 @@ private long toMicros(long value, ChronoUnit unit) { @TestOnly public long totalBufferedBytes() { long total = 0; + for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { + QwpTableBuffer retired = retiredTableBuffers.getQuick(i); + if (retired != null) { + total += retired.getBufferedBytes(); + } + } ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); @@ -5390,6 +5927,15 @@ boolean isAborted() { : (cursorSendLoop == null ? closed : !cursorSendLoop.isRunning()); } + boolean requiresSchema() { + return schemaRequired; + } + + @Override + public void requireSchema() { + schemaRequired = true; + } + @Override public WebSocketClient reconnect() { return buildAndConnect(this, null); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 81d59d28f..11c9df07a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -24,7 +24,11 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.std.LongList; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Utf8SequenceObjHashMap; @@ -32,6 +36,9 @@ import io.questdb.client.std.str.Utf8s; import org.jetbrains.annotations.TestOnly; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; /** @@ -63,6 +70,9 @@ */ public class WebSocketResponse { + public static final int SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL = 0xC0; + public static final int SCHEMA_FEEDBACK_MODE_MASK = 0xC0; + public static final int SCHEMA_FEEDBACK_MODE_UPDATES = 0x80; public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; public static final int MIN_DURABLE_ACK_SIZE = 3; // status + tableCount public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length @@ -102,9 +112,12 @@ public class WebSocketResponse { private final Utf8SequenceObjHashMap tableNameCache = new Utf8SequenceObjHashMap<>(); private final ObjList tableNames = new ObjList<>(); private final LongList tableSeqTxns = new LongList(); + private final ObjList schemaUpdateTableNames = new ObjList<>(); + private final ObjList schemaUpdates = new ObjList<>(); private String errorMessage; private int errorMessageUtf8Length; private long sequence; + private boolean schemaInvalidation; private byte status; public WebSocketResponse() { @@ -249,6 +262,26 @@ public long getTableSeqTxn(int index) { return tableSeqTxns.getQuick(index); } + public int getSchemaUpdateCount() { + return schemaUpdates.size(); + } + + public QwpSchemaResponse getSchemaUpdate(int index) { + return schemaUpdates.getQuick(index); + } + + public String getSchemaUpdateTableName(int index) { + return schemaUpdateTableNames.getQuick(index); + } + + public boolean hasSchemaUpdates() { + return schemaUpdates.size() > 0; + } + + public boolean isSchemaInvalidation() { + return schemaInvalidation; + } + /** * Returns true when this is a per-table durable-upload ACK (STATUS_DURABLE_ACK). */ @@ -271,14 +304,38 @@ public boolean isSuccess() { * @return true if successfully parsed, false if not enough data */ public boolean readFrom(long ptr, int length) { + return readFrom(ptr, length, false); + } + + /** + * Reads a response, accepting schema feedback only on a connection that + * negotiated schema support during its WebSocket upgrade. + */ + public boolean readFrom(long ptr, int length, boolean schemaNegotiated) { tableNames.clear(); tableSeqTxns.clear(); + schemaUpdateTableNames.clear(); + schemaUpdates.clear(); + schemaInvalidation = false; if (length < 1) { return false; } - status = Unsafe.getUnsafe().getByte(ptr); + int wireStatus = Unsafe.getUnsafe().getByte(ptr) & 0xff; + int feedbackMode = wireStatus & SCHEMA_FEEDBACK_MODE_MASK; + if (feedbackMode == 0x40 || (feedbackMode != 0 && !schemaNegotiated)) { + return false; + } + if (feedbackMode == SCHEMA_FEEDBACK_MODE_UPDATES && length > QwpSchemaProtocol.MAX_FRAME_SIZE) { + return false; + } + byte baseStatus = (byte) (wireStatus & ~SCHEMA_FEEDBACK_MODE_MASK); + if (baseStatus == STATUS_DURABLE_ACK && feedbackMode != 0) { + return false; + } + status = baseStatus; + int baseEnd; if (status == STATUS_OK) { if (length < MIN_OK_RESPONSE_SIZE) { @@ -287,7 +344,12 @@ public boolean readFrom(long ptr, int length) { sequence = Unsafe.getUnsafe().getLong(ptr + 1); errorMessage = null; errorMessageUtf8Length = -1; - return readTableEntries(ptr + 9, length - 9); + baseEnd = readTableEntries(ptr + 9, length - 9, feedbackMode != 0); + if (baseEnd < 0) { + return false; + } + baseEnd += 9; + return readFeedback(ptr, length, baseEnd, feedbackMode); } if (status == STATUS_DURABLE_ACK) { @@ -297,7 +359,7 @@ public boolean readFrom(long ptr, int length) { sequence = -1; errorMessage = null; errorMessageUtf8Length = -1; - return readTableEntries(ptr + 1, length - 1); + return readTableEntries(ptr + 1, length - 1, false) >= 0; } // Error response @@ -322,7 +384,8 @@ public boolean readFrom(long ptr, int length) { errorMessage = null; errorMessageUtf8Length = 0; } - return true; + baseEnd = offset + msgLen; + return readFeedback(ptr, length, baseEnd, feedbackMode); } /** @@ -391,15 +454,15 @@ public int writeTo(long ptr) { // entries, empty table names, or trailing garbage. On false, tableNames / // tableSeqTxns may hold partial state, but the caller (readFrom) clears // both lists at the start of every call so partial state never leaks. - private boolean readTableEntries(long ptr, int remaining) { + private int readTableEntries(long ptr, int remaining, boolean allowTrailing) { if (remaining < 2) { - return false; + return -1; } int tableCount = Unsafe.getUnsafe().getShort(ptr) & 0xFFFF; int offset = 2; for (int i = 0; i < tableCount; i++) { if (remaining < offset + 2) { - return false; + return -1; } int nameLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF; offset += 2; @@ -407,7 +470,7 @@ private boolean readTableEntries(long ptr, int remaining) { // table name is never zero bytes, and accepting empty names would // let a misbehaving server poison the per-table tracker with "" entries. if (nameLen == 0 || remaining < offset + nameLen + 8) { - return false; + return -1; } long nameLo = ptr + offset; long nameHi = nameLo + nameLen; @@ -417,7 +480,87 @@ private boolean readTableEntries(long ptr, int remaining) { tableNames.add(internTableName(nameLo, nameHi)); tableSeqTxns.add(seqTxn); } - return remaining == offset; + return allowTrailing || remaining == offset ? offset : -1; + } + + private boolean readFeedback(long ptr, int length, int offset, int mode) { + if (mode == 0) { + return offset == length; + } + if (mode == SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL) { + if (offset != length) { + return false; + } + schemaInvalidation = true; + return true; + } + if (length - offset < 2) { + return false; + } + int count = Unsafe.getUnsafe().getShort(ptr + offset) & 0xffff; + offset += 2; + if (count == 0) { + return false; + } + // Each entry needs nameLen + one name byte + payloadLen + the shortest + // schema payload. Reject hostile counts before sizing collections. + if (count > (length - offset) / 17) { + return false; + } + ObjList names = new ObjList<>(count); + ObjList responses = new ObjList<>(count); + LowerCaseCharSequenceIntHashMap seen = new LowerCaseCharSequenceIntHashMap(count); + try { + for (int i = 0; i < count; i++) { + if (length - offset < 2) { + return false; + } + int nameLength = Unsafe.getUnsafe().getShort(ptr + offset) & 0xffff; + offset += 2; + if (nameLength == 0 || nameLength > QwpSchemaProtocol.MAX_NAME_UTF8_LENGTH || length - offset < nameLength + 4) { + return false; + } + String name = decodeStrictUtf8(ptr + offset, nameLength); + if (!TableUtils.isValidTableName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH) || !seen.put(name, i)) { + return false; + } + offset += nameLength; + long payloadLength = Unsafe.getUnsafe().getInt(ptr + offset) & 0xffffffffL; + offset += 4; + if (payloadLength > QwpSchemaProtocol.MAX_FRAME_SIZE || payloadLength > length - offset) { + return false; + } + QwpSchemaResponse response = QwpSchemaProtocol.decodeFeedbackPayload(ptr + offset, (int) payloadLength); + offset += (int) payloadLength; + names.add(name); + responses.add(response); + } + } catch (IllegalArgumentException e) { + return false; + } + if (offset != length) { + return false; + } + for (int i = 0; i < names.size(); i++) { + schemaUpdateTableNames.add(names.getQuick(i)); + schemaUpdates.add(responses.getQuick(i)); + } + return true; + } + + private static String decodeStrictUtf8(long address, int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("invalid UTF-8 feedback table name", e); + } } private String internTableName(long lo, long hi) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index facd872bb..b6f2a17e8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpCredentialUnavailableException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; @@ -407,6 +408,10 @@ public static long dynamicCredentialAuthDwellNanos(long reconnectMaxDurationMill * {@link #outcome} has been set to FAILED or STOPPED */ public WebSocketClient connectWithDurableAckRetry() { + return connectWithDurableAckRetry(null); + } + + private WebSocketClient connectWithDurableAckRetry(CursorSendEngine schemaEngine) { // run() already set runnerThread; setting it again here is a no-op // on that path but wires up direct callers so requestStop() // can unpark them too. @@ -477,7 +482,13 @@ public WebSocketClient connectWithDurableAckRetry() { // to the deadline (which would otherwise busy-loop once past it). boolean boundedByBudget = false; try { - return clientFactory.reconnect(); + WebSocketClient connected = clientFactory.reconnect(); + if (connected != null && schemaEngine != null && schemaEngine.requiresSchema() + && !connected.isQwpSchemaEnabled()) { + connected.close(); + throw new QwpSchemaCapabilityMismatchException(); + } + return connected; } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // A non-421 upgrade reject, and a 401/403 against a CONSTANT credential, are genuinely // non-retriable across the cluster: waiting will not fix them, so quarantine immediately @@ -589,6 +600,16 @@ public WebSocketClient connectWithDurableAckRetry() { slotPath, roleRejectAttempts); lastReplicaWarnNanos = nowWarn; } + } catch (QwpSchemaCapabilityMismatchException e) { + // Extended bytes cannot be downgraded, discarded or quarantined. + // Keep the slot and retry until a supporting endpoint returns. + capabilityGapAttempts = 0; + capabilityGapElapsedNanos = 0L; + lastCapabilityGapNanos = 0L; + firstDynamicCredentialAuthFailureNanos = 0L; + lastErrorMessage = e.getMessage(); + LOG.warn("drainer slot {}: schema framing unavailable, retaining backlog and retrying", + slotPath); } catch (QwpDurableAckMismatchException e) { // Genuine cluster-wide durable-ack CAPABILITY gap: a server // upgraded but does not advertise durable ack. Unlike a role @@ -1051,6 +1072,9 @@ public void run() { return; } engineForTesting = engine; + if (engine.requiresSchema()) { + clientFactory.requireSchema(); + } if (logicalSlotLock != null) { logicalSlotLock.close(); logicalSlotLock = null; @@ -1072,7 +1096,7 @@ public void run() { // poll of a partially-drained slot read as progress and hand back a budget the initial connect // had legitimately spent. ackProgressWatermark = engine.ackedFsn(); - client = connectWithDurableAckRetry(); + client = connectWithDurableAckRetry(engine); if (client == null) { // outcome already set (FAILED or STOPPED); markFailed sentinel // already dropped on the FAILED path. @@ -1181,7 +1205,8 @@ public void run() { if (t.getCause() instanceof Error) { throw (Error) t.getCause(); } - if (loop.capabilityGapTerminal() != null || loop.authTerminal() != null) { + if (loop.capabilityGapTerminal() != null + || loop.authTerminal() != null) { // The I/O thread publishes a durable ack before it latches a later terminal. // That publication can land after the poll at the top of this iteration but // before checkError() observes the terminal. Re-read here so recycling the wire @@ -1243,7 +1268,7 @@ public void run() { } catch (Throwable ignored) { } loop = null; - client = connectWithDurableAckRetry(); + client = connectWithDurableAckRetry(engine); if (client == null) { // outcome already set (FAILED after budget // exhaustion, or STOPPED); sentinel handled. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 66a0635ea..21ac445b3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; @@ -174,6 +175,7 @@ public final class CursorSendEngine implements QuietCloseable { // because both producer seeding and every recycled send loop need the same // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. private final RecoveredFrameAnalysis recoveredFrameAnalysis; + private volatile boolean requiresSchema; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -820,6 +822,8 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.watermark = watermarkInProgress; this.persistedSymbolDict = persistedDictInProgress; this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; + this.requiresSchema = recoveredFrameAnalysisInProgress != null + && recoveredFrameAnalysisInProgress.requiresSchema(); } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -916,6 +920,7 @@ MmapSegment pinActiveSegment() { * is wedged" failures (server down, slow disk, etc.) from the user. */ public long appendBlocking(long payloadAddr, int payloadLen) { + latchSchemaRequirement(payloadAddr, payloadLen); long fsn = ring.appendOrFsn(payloadAddr, payloadLen); if (fsn >= 0) return fsn; if (fsn == SegmentRing.PAYLOAD_TOO_LARGE) { @@ -959,6 +964,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) { * {@code SegmentRing.BACKPRESSURE_*} / {@code PAYLOAD_*} sentinels. */ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos) { + latchSchemaRequirement(payloadAddr, payloadLen); long fsn = ring.appendOrFsn(payloadAddr, payloadLen); if (fsn >= 0) { return fsn; @@ -983,6 +989,27 @@ public void checkDurability() { ring.checkDurability(); } + public boolean requiresSchema() { + return requiresSchema; + } + + public void requireSchema() { + requiresSchema = true; + } + + private void latchSchemaRequirement(long payloadAddr, int payloadLen) { + if (!requiresSchema + && payloadAddr != 0 + && payloadLen >= QwpConstants.HEADER_SIZE + && io.questdb.client.std.Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE + && (io.questdb.client.std.Unsafe.getUnsafe().getByte( + payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA) != 0) { + // Publish the capability requirement before ring.append publishes + // the frame to the I/O consumer. + requiresSchema = true; + } + } + @Override public void close() { close(true); @@ -1882,10 +1909,17 @@ public long recoveredOrphanTipFsn() { * server ACKs */ public boolean retireRecoveredOrphanTailIfReady() { + return retireRecoveredOrphanTailIfReady(false); + } + + public boolean retireRecoveredOrphanTailIfReady(boolean schemaConfirmed) { long orphanTip = recoveredOrphanTipFsn; if (orphanTip < 0L) { return true; } + if (requiresSchema && !schemaConfirmed) { + return false; + } long orphanStart = recoveredCommitBoundaryFsn + 1L; if (ackedFsn() < orphanStart - 1L) { return false; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6643bf036..b47870299 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.LineSenderServerException; +import io.questdb.client.LineSenderSchemaException; import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderError; import io.questdb.client.cutlass.http.client.WebSocketClient; @@ -35,11 +36,14 @@ import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpCredentialUnavailableException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; import io.questdb.client.std.MemoryTag; @@ -271,6 +275,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final CursorSendEngine engine; private final long parkNanos; private final ReconnectPolicy reconnectPolicy; + private final QwpSchemaCoordinator schemaCoordinator = new QwpSchemaCoordinator(); // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -919,6 +924,16 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // hands null and lets the I/O thread connect — hasEverConnected // stays false until swapClient sees its first success. this.hasEverConnected = client != null; + if (engine.requiresSchema()) { + if (reconnectFactory != null) { + reconnectFactory.requireSchema(); + } + } else if (client != null && client.isQwpSchemaEnabled()) { + engine.requireSchema(); + if (reconnectFactory != null) { + reconnectFactory.requireSchema(); + } + } // Adopt the engine's recovered orphaned-deferred-tail range (if any). // See the field docs on orphanSkipStartFsn/orphanSkipTipFsn; the // BackgroundDrainer path builds its loop through this same @@ -1201,6 +1216,35 @@ public void checkError() { } } + /** + * Resolves a schema through this loop's I/O thread, returning a KNOWN or MISSING response. + * This producer-side API permits one outstanding lookup. The timeout covers the entire + * operation, including connection and I/O; zero permits only an existing cache hit. + * Failures are reported as {@link LineSenderSchemaException}. This does not mutate rows, + * store-and-forward state, or acknowledgment watermarks. + */ + public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.resolve(tableName, timeoutMillis, false, ioThread); + } + + public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMillis, boolean[] freshOut) { + if (freshOut == null || freshOut.length == 0) { + throw new IllegalArgumentException("freshOut must contain one element"); + } + return schemaCoordinator.resolve(tableName, timeoutMillis, false, ioThread, freshOut); + } + + /** + * Evicts the named cache entry at admission and resolves it again through the I/O thread. + * This producer-side API permits one outstanding lookup. The timeout covers the entire + * operation, including connection and I/O, and must leave time for a request. It returns + * a KNOWN or MISSING response and reports failures as {@link LineSenderSchemaException}. + * This does not mutate rows, store-and-forward state, or acknowledgment watermarks. + */ + public QwpSchemaResponse refreshSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.resolve(tableName, timeoutMillis, true, ioThread); + } + /** * The typed rotating-credential auth terminal (401/403), or {@code null} if the loop's terminal * (if any) is a different failure class. Non-null only after {@link #checkError()} started @@ -1259,6 +1303,7 @@ public synchronized void close() { // is false and start's ioLoop will exit immediately) or entirely // after — the latch await is only skipped when the loop never ran. running = false; + schemaCoordinator.close(); Thread t = ioThread; // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets // freed on ioLoop's exit path. When t == null the loop never ran (start() @@ -1534,6 +1579,41 @@ public boolean hasEverConnected() { return hasEverConnected; } + /** + * Waits for the first completed WebSocket upgrade and returns whether that + * handshake selected schema mode. A failed or still-pending handshake is + * never interpreted as a legacy peer. + */ + public boolean awaitInitialSchemaMode(long timeoutMillis) { + long timeoutNanos = timeoutMillis > Long.MAX_VALUE / 1_000_000L + ? Long.MAX_VALUE : timeoutMillis * 1_000_000L; + long start = System.nanoTime(); + while (!hasEverConnected) { + checkError(); + if (!running) { + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation stopped before the first successful connection"); + } + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation was interrupted"); + } + if (timeoutNanos - (System.nanoTime() - start) <= 0) { + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation timed out before the first successful connection"); + } + LockSupport.parkNanos(Math.min(50_000L, + timeoutNanos - (System.nanoTime() - start))); + } + // swapClient/constructor publishes the sticky engine requirement before + // hasEverConnected becomes visible. Read the authority, not client state. + return engine.requiresSchema(); + } + public boolean isRunning() { return running; } @@ -1584,6 +1664,10 @@ public synchronized void start() { throw new IllegalStateException("already started"); } running = true; + if (engine.requiresSchema() && client != null && !client.isQwpSchemaEnabled()) { + running = false; + throw new QwpSchemaCapabilityMismatchException(); + } // Position the cursor at the first unsent FSN before spinning the // I/O thread. For a fresh sender, ackedFsn=-1 → start at FSN 0, // which lands on the (empty) initial active — same as the prior @@ -1732,6 +1816,10 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM if (!running) { return; } + WebSocketClient failedClient = client; + if (failedClient != null) { + schemaCoordinator.connectionLost(failedClient); + } if (reconnectFactory == null) { recordFatal(initial); return; @@ -1788,6 +1876,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { + if (engine.requiresSchema()) { + reconnectFactory.requireSchema(); + } WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { @@ -1884,6 +1975,11 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, attempts, e.getMessage()); lastLogNanos = now; } + } catch (QwpSchemaCapabilityMismatchException e) { + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + LOG.warn("schema framing unavailable during {}; retaining backlog and retrying: {}", + phase, e.getMessage()); } catch (QwpDurableAckMismatchException e) { if (endpointPolicyFailureIsTerminal()) { // Orphans hand a capability gap back to BackgroundDrainer's @@ -2376,7 +2472,10 @@ private void ioLoop() { attemptInitialConnect(); } while (running) { - boolean didWork = trySendOne(); + boolean didWork = trySendSchemaRequest(); + if (trySendOne()) { + didWork = true; + } // 1. Try to send next frame(s). // 2. Try to receive ACKs. if (tryReceiveAcks()) { @@ -2430,6 +2529,7 @@ private void ioLoop() { } } } finally { + schemaCoordinator.close(); // Release native-segment lifetime before publishing I/O-thread // completion or running delegated engine cleanup. releaseSendingSegment(); @@ -2549,6 +2649,7 @@ private void positionCursorInSegment(MmapSegment seg, long targetFsn) { * every rethrow delivers the same instance. */ private void recordFatal(Throwable t) { + schemaCoordinator.close(); if (terminalError == null) { terminalError = t instanceof LineSenderException ? (LineSenderException) t @@ -2646,8 +2747,23 @@ private void snapshotReplayTarget() { * {@link #trySendOne} call replays the first unacked frame. */ private void swapClient(WebSocketClient newClient) { + if (engine.requiresSchema() && !newClient.isQwpSchemaEnabled()) { + try { + newClient.close(); + } catch (Throwable ignored) { + // best-effort + } + throw new QwpSchemaCapabilityMismatchException(); + } + if (newClient.isQwpSchemaEnabled()) { + engine.requireSchema(); + if (reconnectFactory != null) { + reconnectFactory.requireSchema(); + } + } WebSocketClient old = this.client; this.client = newClient; + schemaCoordinator.clearCache(); this.lastReconnectError = null; if (old != null) { try { @@ -3390,12 +3506,53 @@ private boolean tryReceiveAcks() { return any; } + private boolean trySendSchemaRequest() { + if (!schemaCoordinator.hasPendingRequest()) { + return false; + } + WebSocketClient activeClient = client; + if (activeClient == null) { + return false; + } + QwpSchemaCoordinator.Request request = schemaCoordinator.requestToSend(activeClient); + if (request == null) { + return false; + } + if (!activeClient.isQwpSchemaEnabled()) { + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + "server did not negotiate schema lookup support"); + return true; + } + long remainingNanos = request.remainingNanos(); + if (remainingNanos <= 0) { + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema lookup timed out before send"); + return true; + } + int timeoutMillis = (int) Math.min(Integer.MAX_VALUE, + 1L + (remainingNanos - 1L) / 1_000_000L); + try { + activeClient.sendBinary(request.message, timeoutMillis); + return true; + } catch (Throwable t) { + if (t instanceof Error) { + throw (Error) t; + } + fail(t); + return true; + } + } + /** * Returns true if at least one frame was sent (caller skips the park). * Bounded: sends at most one frame per call so the ACK side gets * scheduling fairness. */ private boolean trySendOne() { + if (engine.requiresSchema() && (client == null || !client.isQwpSchemaEnabled())) { + fail(new QwpSchemaCapabilityMismatchException()); + return false; + } if (orphanSkipTipFsn >= 0 && fsnAtZero + nextWireSeq >= orphanSkipStartFsn) { // The send cursor reached the orphaned deferred tail. Its frames // belong to an aborted transaction and must never be transmitted @@ -3485,6 +3642,14 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // The producer latches the requirement before publishing an extended + // frame. Re-check after observing publication so an append racing the + // early guard cannot expose either its legacy prefix or the extended + // frame to an already-connected legacy peer. + if (engine.requiresSchema() && (client == null || !client.isQwpSchemaEnabled())) { + fail(new QwpSchemaCapabilityMismatchException()); + return false; + } // Torn-dictionary guard. sentDictCount is this loop's model of how many ids the // CURRENT server has been told about. A frame whose delta starts ABOVE that // coverage references ids the server was never given, and the server now rejects @@ -3592,7 +3757,8 @@ private boolean tryRetireOrphanTail() { if (orphanSkipTipFsn < 0) { return true; } - if (!engine.retireRecoveredOrphanTailIfReady()) { + if (!engine.retireRecoveredOrphanTailIfReady( + client != null && client.isQwpSchemaEnabled())) { return false; } orphanSkipStartFsn = -1L; @@ -3631,6 +3797,9 @@ public enum ReconnectPolicy { public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; + default void requireSchema() { + } + /** * Whether this factory re-derives its {@code Authorization} header from a caller-supplied token * provider on every attempt, rather than presenting a constant captured once. @@ -3862,11 +4031,43 @@ boolean isDurableUnder(CharSequenceLongHashMap watermarks) { private final class ResponseHandler implements WebSocketFrameHandler { @Override public void onBinaryMessage(long payloadPtr, int payloadLen) { - if (!response.readFrom(payloadPtr, payloadLen)) { + if (payloadLen >= Integer.BYTES + && Unsafe.getUnsafe().getInt(payloadPtr) == QwpConstants.MAGIC_MESSAGE) { + WebSocketClient responseClient = client; + final long requestId; + try { + if (responseClient == null || !responseClient.isQwpSchemaEnabled()) { + throw new IllegalArgumentException("schema control response was not negotiated"); + } + requestId = QwpSchemaProtocol.peekResponseRequestId(payloadPtr, payloadLen); + } catch (RuntimeException e) { + fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); + return; + } + if (!schemaCoordinator.isLiveResponse(responseClient, requestId)) { + return; + } + final QwpSchemaResponse schema; + try { + schema = QwpSchemaProtocol.decodeResponse(payloadPtr, payloadLen); + } catch (RuntimeException e) { + if (!schemaCoordinator.isLiveResponse(responseClient, requestId)) { + return; + } + fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); + return; + } + schemaCoordinator.completeResponse(responseClient, schema); + return; + } + if (!response.readFrom(payloadPtr, payloadLen, client != null && client.isQwpSchemaEnabled())) { fail(new LineSenderException( "Invalid ACK response payload [length=" + payloadLen + ']')); return; } + if (response.hasSchemaUpdates() || response.isSchemaInvalidation()) { + schemaCoordinator.applyFeedback(response); + } long wireSeq = response.getSequence(); if (response.isSuccess()) { // Same sanity clamp as legacy: don't trust an ACK beyond diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java new file mode 100644 index 000000000..1c54a2900 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -0,0 +1,288 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.std.Chars; + +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.locks.LockSupport; + +import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; +import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; +import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; + +final class QwpSchemaCoordinator { + static final int MAX_CACHE_ENTRIES = 1_000_000; + private LinkedHashMap cache; + private volatile boolean closed; + private volatile boolean hasPendingRequest; + private long nextRequestId = 1; + private Request request; + + QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean refresh, Thread ioThread) { + return resolve(tableName, timeoutMillis, refresh, ioThread, null); + } + + QwpSchemaResponse resolve( + CharSequence tableName, + long timeoutMillis, + boolean refresh, + Thread ioThread, + boolean[] freshOut + ) { + final long startNanos = System.nanoTime(); + if (timeoutMillis < 0) { + throw new IllegalArgumentException("timeoutMillis must be non-negative"); + } + if (tableName == null || tableName.length() == 0 + || tableName.length() > QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH) { + throw new IllegalArgumentException("table name must contain 1 to " + + QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + " UTF-16 units"); + } + final String key = normalize(tableName); + final long timeoutNanos = timeoutNanos(timeoutMillis); + synchronized (this) { + if (closed) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema coordinator is closed"); + } + if (!refresh) { + QwpSchemaResponse cached = cache == null ? null : cache.get(key); + if (cached != null) { + if (freshOut != null) { + freshOut[0] = false; + } + return cached; + } + } else { + remove(key); + } + if (timeoutNanos - (System.nanoTime() - startNanos) <= 0) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); + } + if (request != null) { + throw failure(SCHEMA_UNAVAILABLE, key, "another schema lookup is already in progress"); + } + if (nextRequestId <= 0 || nextRequestId == Long.MAX_VALUE) { + throw failure(UNSUPPORTED_FEATURE, key, "schema request id space is exhausted"); + } + long id = nextRequestId; + if (freshOut != null) { + freshOut[0] = true; + } + byte[] message = QwpSchemaProtocol.encodeDescribe(id, tableName); + nextRequestId++; + Request own = new Request(id, key, startNanos, timeoutNanos, message); + request = own; + hasPendingRequest = true; + LockSupport.unpark(ioThread); + while (!own.done) { + try { + await(own, key); + } catch (LineSenderSchemaException e) { + if (request == own) { + request = null; + hasPendingRequest = false; + notifyAll(); + } + throw e; + } + } + if (own.error != null) { + throw own.error; + } + return own.response; + } + } + + synchronized Request requestToSend(WebSocketClient client) { + if (request == null || request.done || request.sentClient != null) { + return null; + } + if (request.remainingNanos() <= 0) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema lookup timed out")); + return null; + } + request.sentClient = client; + return request; + } + + boolean hasPendingRequest() { + return hasPendingRequest; + } + + synchronized boolean isLiveResponse(WebSocketClient client, long requestId) { + Request current = request; + return current != null && !current.done && current.sentClient == client + && current.id == requestId && current.remainingNanos() > 0; + } + + synchronized void completeResponse(WebSocketClient client, QwpSchemaResponse response) { + Request current = request; + if (current == null || current.done || current.sentClient != client + || current.id != response.getRequestId()) { + return; + } + if (current.remainingNanos() <= 0) { + complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema lookup timed out")); + return; + } + switch (response.getResult()) { + case QwpSchemaProtocol.RESULT_KNOWN: + case QwpSchemaProtocol.RESULT_MISSING: + put(current.key, response); + complete(current, response, null); + return; + case QwpSchemaProtocol.RESULT_DENIED: + remove(current.key); + complete(current, null, failure(ACCESS_DENIED, current.key, "schema access denied")); + return; + case QwpSchemaProtocol.RESULT_UNAVAILABLE: + remove(current.key); + complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema is unavailable")); + return; + case QwpSchemaProtocol.RESULT_TOO_LARGE: + default: + remove(current.key); + complete(current, null, failure(UNSUPPORTED_FEATURE, current.key, "schema response is too large or unsupported")); + } + } + + synchronized void fail(Request own, LineSenderSchemaException.Reason reason, String detail) { + if (request == own && !own.done) { + complete(own, null, failure(reason, own.key, detail)); + } + } + + void applyFeedback(WebSocketResponse feedback) { + if (feedback.isSchemaInvalidation()) { + clearCache(); + } + for (int i = 0, n = feedback.getSchemaUpdateCount(); i < n; i++) { + QwpSchemaResponse schema = feedback.getSchemaUpdate(i); + String key = normalize(feedback.getSchemaUpdateTableName(i)); + synchronized (this) { + if (closed) { + return; + } + if (schema.getResult() == QwpSchemaProtocol.RESULT_KNOWN + || schema.getResult() == QwpSchemaProtocol.RESULT_MISSING) { + put(key, schema); + } else { + remove(key); + } + } + } + } + + synchronized void connectionLost(WebSocketClient client) { + if (request != null && (request.sentClient == null || request.sentClient == client)) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "connection lost during schema lookup")); + } + } + + synchronized void clearCache() { + if (!closed) { + cache = null; + } + } + + synchronized void close() { + closed = true; + cache = null; + if (request != null) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema coordinator is closed")); + } + notifyAll(); + } + + private static long timeoutNanos(long timeoutMillis) { + return timeoutMillis > Long.MAX_VALUE / 1_000_000L ? Long.MAX_VALUE : timeoutMillis * 1_000_000L; + } + + private static LineSenderSchemaException failure(LineSenderSchemaException.Reason reason, String table, String detail) { + return new LineSenderSchemaException(reason, "schema lookup failed [reason=" + reason + + ", table=" + table + ", detail=" + detail + ']'); + } + + private static String normalize(CharSequence tableName) { + return Chars.toLowerCase(tableName); + } + + private void await(Request request, String key) { + long remaining = request.remainingNanos(); + if (remaining <= 0) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); + } + try { + long millis = remaining / 1_000_000L; + int nanos = (int) (remaining % 1_000_000L); + wait(millis, nanos); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup was interrupted"); + } + } + + private void complete(Request own, QwpSchemaResponse response, LineSenderSchemaException error) { + own.response = response; + own.error = error; + own.done = true; + if (request == own) { + request = null; + hasPendingRequest = false; + } + notifyAll(); + } + + private void put(String key, QwpSchemaResponse schema) { + if (cache == null) { + cache = new LinkedHashMap<>(); + } + cache.put(key, schema); + while (cache.size() > MAX_CACHE_ENTRIES) { + Iterator> iterator = cache.entrySet().iterator(); + iterator.next(); + iterator.remove(); + } + } + + private void remove(String key) { + if (cache != null) { + cache.remove(key); + } + } + + static final class Request { + final long startNanos; + final long timeoutNanos; + final long id; + final String key; + final byte[] message; + boolean done; + LineSenderSchemaException error; + QwpSchemaResponse response; + WebSocketClient sentClient; + + Request(long id, String key, long startNanos, long timeoutNanos, byte[] message) { + this.id = id; + this.key = key; + this.startNanos = startNanos; + this.timeoutNanos = timeoutNanos; + this.message = message; + } + + long remainingNanos() { + return timeoutNanos - (System.nanoTime() - startNanos); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index 6a7f70df3..c4fe33d8b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -65,6 +65,7 @@ final class RecoveredFrameAnalysis implements QuietCloseable { private int runningRawCount; private int runningRawLen; private long runningCoverage; + private boolean requiresSchema; RecoveredFrameAnalysis(int baseline, long ackedFsn) { this.baseline = baseline; @@ -81,6 +82,11 @@ void accept(long fsn, long payload, int payloadLen) { byte flags = isQwp ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS) : 0; + if (fsn > ackedFsn && isQwp && (flags & QwpConstants.FLAG_SCHEMA) != 0) { + // Conservative by design: retained deferred tails may still be + // replay-owned until the engine's orphan-tail retirement runs. + requiresSchema = true; + } if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) { foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen); } @@ -205,6 +211,10 @@ int rawCapacity() { return rawCapacity; } + boolean requiresSchema() { + return requiresSchema; + } + /** * Releases the cached suffix after a foreground producer and its one send * loop have both consumed it. Recovery metadata remains available, but the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 074cf0e3f..d4c12a15d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -41,6 +41,8 @@ public final class QwpConstants { * but skips the commit until a subsequent message without this flag. */ public static final byte FLAG_DEFER_COMMIT = 0x01; + /** Flag bit: table headers carry schema identity information. */ + public static final byte FLAG_SCHEMA = 0x02; /** * Flag bit: Delta symbol dictionary encoding enabled. * When set, symbol columns use global IDs and send only new dictionary entries. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java new file mode 100644 index 000000000..77ff7bd5e --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -0,0 +1,1367 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cairo.MicrosTimestampDriver; +import io.questdb.client.cairo.NanosTimestampDriver; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; +import io.questdb.client.std.NumericException; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.bytes.DirectByteSlice; +import io.questdb.client.std.fastdouble.FastDoubleParser; +import io.questdb.client.std.fastdouble.FastFloatParser; +import io.questdb.client.std.str.StringSink; +import io.questdb.client.std.str.Utf8StringSink; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; +import static io.questdb.client.LineSenderSchemaException.Reason.INVALID_VALUE; +import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; +import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; + +/** + * Converts supported input values against one immutable server schema result + * and appends them to a caller-owned {@link QwpTableBuffer}. A confirmed + * missing table, or a column absent from a known schema, is inferred from the + * first effective supported input value. + * + *

This binding does not own the buffer or its row lifecycle and is not + * thread-safe. The caller completes rows, encodes data and closes the buffer. + * After a setter failure, the caller must invoke both + * {@link QwpTableBuffer#cancelCurrentRow()} and + * {@link QwpTableBuffer#rollbackUncommittedColumns()} before continuing or + * completing another row. {@link QwpTableBuffer#reset()} preserves the binding; + * {@link QwpTableBuffer#clear()} invalidates it.

+ * + *

The caller must encode the buffer with the matching symbol-dictionary + * family: local dictionaries for ownerless buffers, or the owning sender's + * corresponding global dictionary for sender-backed buffers.

+ */ +public final class QwpSchemaBinding { + private static final int FIRST_DASH = 8; + private static final int SECOND_DASH = 13; + private static final int THIRD_DASH = 18; + private static final int FOURTH_DASH = 23; + private static final int UUID_LENGTH = 36; + + private final QwpTableBuffer buffer; + private final LowerCaseCharSequenceIntHashMap columns = new LowerCaseCharSequenceIntHashMap(); + private final QwpSchemaResponse schema; + private final String tableName; + private Decimal256 decimalTextScratch; + private int[] floatingTextExponentScratch; + private StringSink floatingTextSink; + private StringSink numericTextSink; + private Utf8StringSink timestampTextSink; + private StringSink uuidTextSink; + + /** + * Attaches {@code schema} to an empty, unbound caller-owned buffer. The + * caller must correlate a + * positive request id with its DESCRIBE table, or obtain request-id-zero + * schema from the matching named write-feedback entry. + */ + public QwpSchemaBinding(QwpTableBuffer buffer, QwpSchemaResponse schema) { + if (buffer == null) { + throw new NullPointerException("buffer"); + } + String tableName = buffer.getTableName(); + if (tableName == null || !TableUtils.isValidTableName(tableName, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { + throw new IllegalArgumentException("invalid table name"); + } + if (schema == null) { + throw new NullPointerException("schema"); + } + this.tableName = tableName; + this.schema = schema; + requireUsableSchema(); + for (int i = 0, n = schema.getColumnCount(); i < n; i++) { + columns.put(schema.getColumnName(i), i); + } + this.buffer = buffer; + buffer.attachSchemaBinding(this); + } + + /** + * Appends an owned copy of a BINARY value to an exact BINARY target. + * The caller retains the class-level row-cancellation and uncommitted-column rollback + * responsibility on failure. + */ + public QwpSchemaBinding binaryColumn(CharSequence name, byte[] value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + if (value == null) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary value cannot be null"); + } + column.addBinary(value); + return this; + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal64 value) { + return decimalColumn(name, value, "DECIMAL64", QwpConstants.TYPE_DECIMAL64); + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal128 value) { + return decimalColumn(name, value, "DECIMAL128", QwpConstants.TYPE_DECIMAL128); + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal256 value) { + return decimalColumn(name, value, "DECIMAL256", QwpConstants.TYPE_DECIMAL256); + } + + /** Appends an owned copy of the bytes addressed by {@code slice}. */ + public QwpSchemaBinding binaryColumn(CharSequence name, DirectByteSlice slice) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + if (slice == null) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary slice cannot be null"); + } + appendBinary(name, column, slice.ptr(), slice.size()); + return this; + } + + /** + * Appends an owned copy of {@code [ptr, ptr + len)}. The source memory need only remain + * valid for the duration of this call. + */ + public QwpSchemaBinding binaryColumn(CharSequence name, long ptr, long len) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + appendBinary(name, column, ptr, len); + return this; + } + + public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "BOOLEAN"); + int targetType = targetType(index, ColumnType.BOOLEAN); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BOOLEAN", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.BOOLEAN: + column.addBoolean(value); + break; + case ColumnType.BYTE: + column.addByte((byte) (value ? 1 : 0)); + break; + case ColumnType.SHORT: + column.addShort((short) (value ? 1 : 0)); + break; + case ColumnType.INT: + column.addInt(value ? 1 : 0); + break; + case ColumnType.LONG: + column.addLong(value ? 1 : 0); + break; + case ColumnType.FLOAT: + column.addFloat(value ? 1 : 0); + break; + case ColumnType.DOUBLE: + column.addDouble(value ? 1 : 0); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value ? "true" : "false"); + break; + default: + throw unsupported(name, "BOOLEAN", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding doubleColumn(CharSequence name, double value) { + return floatingNumericColumn(name, value, "DOUBLE"); + } + + /** Appends and pins the row's designated timestamp using the schema target precision. */ + public QwpSchemaBinding designatedTimestamp(long value, ChronoUnit unit) { + buffer.requireSchemaBinding(this); + int targetType = schema.getResult() == QwpSchemaProtocol.RESULT_MISSING + ? (unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO) + : schema.getColumnType(requireDesignatedTimestamp("TIMESTAMP")); + if (unit == null) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "timestamp unit is null"); + } + if (!isSupportedTimestampUnit(unit)) { + throw unsupported(null, "TIMESTAMP", targetType, "timestamp unit is not supported: " + unit); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_MICRO + ? MicrosTimestampDriver.INSTANCE.from(value, unit) + : NanosTimestampDriver.INSTANCE.from(value, unit); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + designatedColumn(targetType, "TIMESTAMP").addLong(converted); + return this; + } + + /** Appends and pins an Instant as the row's designated timestamp. */ + public QwpSchemaBinding designatedTimestamp(Instant value) { + buffer.requireSchemaBinding(this); + int targetType = schema.getResult() == QwpSchemaProtocol.RESULT_MISSING + ? ColumnType.TIMESTAMP_MICRO + : schema.getColumnType(requireDesignatedTimestamp("TIMESTAMP")); + if (value == null) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "timestamp value is null"); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_MICRO + ? instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L) + : instantTimestamp(value, 1_000_000_000L, value.getNano()); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + designatedColumn(targetType, "TIMESTAMP").addLong(converted); + return this; + } + + private QwpTableBuffer.ColumnBuffer designatedColumn(int targetType, String inputType) { + byte wireType = timestampWireType(targetType); + if (schema.getResult() == QwpSchemaProtocol.RESULT_MISSING) { + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(""); + if (inferred != null && inferred.getType() != wireType) { + throw unsupported(null, inputType, -1, "inferred designated timestamp type conflict [inferredType=" + + QwpConstants.getTypeName(inferred.getType()) + ']'); + } + } + return buffer.getOrCreateDesignatedTimestampColumn(wireType); + } + + public QwpSchemaBinding floatColumn(CharSequence name, float value) { + return floatingNumericColumn(name, value, "FLOAT"); + } + + public QwpSchemaBinding longColumn(CharSequence name, long value) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "LONG"); + int targetType = targetType(index, ColumnType.LONG); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "LONG", index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType) + && !isTextTarget(targetType) + && !ColumnType.isDecimal(targetType) + && targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO) { + throw unsupported(name, "LONG", targetType, "conversion is not implemented"); + } + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (value == Long.MIN_VALUE) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaLongDecimal(value, targetPrecision, targetScale)) { + throw error(INVALID_VALUE, name, "LONG", targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "LONG", targetType, "decimal value cannot be rescaled exactly"); + } + return this; + } + if (value == Long.MIN_VALUE) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw invalidRange(name, targetType); + } + column.addByte((byte) value); + break; + case ColumnType.SHORT: + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw invalidRange(name, targetType); + } + column.addShort((short) value); + break; + case ColumnType.INT: + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw invalidRange(name, targetType); + } + column.addInt((int) value); + break; + case ColumnType.LONG: + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + column.addLong(value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble((double) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatLong(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatLong(value)); + break; + default: + throw unsupported(name, "LONG", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "STRING"); + int targetType = targetType(index, ColumnType.VARCHAR); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "STRING", index, targetType); + if (column == null) { + return this; + } + if (ColumnType.isGeoHash(targetType)) { + int precision = ColumnType.getGeoHashBits(targetType); + column.initGeoHashPrecision(precision); + if (value == null || value.length() == 0) { + column.addNull(); + } else { + appendStringGeoHash(column, name, value, precision, targetType); + } + return this; + } + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (value == null) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaStringDecimal(value, targetPrecision, targetScale)) { + throw error(INVALID_VALUE, name, "STRING", targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid decimal text"); + } + return this; + } + if (value == null) { + column.addNull(); + } else { + switch (targetType) { + case ColumnType.BYTE: + case ColumnType.SHORT: + case ColumnType.INT: + case ColumnType.LONG: + case ColumnType.FLOAT: + case ColumnType.DOUBLE: + appendStringNumeric(column, name, value, targetType); + break; + case ColumnType.BOOLEAN: + if (value.length() == 1 && value.charAt(0) == '1') { + column.addBoolean(true); + } else if (value.length() == 1 && value.charAt(0) == '0') { + column.addBoolean(false); + } else if (Chars.equalsLowerCaseAscii(value, "true")) { + column.addBoolean(true); + } else if (Chars.equalsLowerCaseAscii(value, "false")) { + column.addBoolean(false); + } else { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid BOOLEAN text"); + } + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value); + break; + case ColumnType.SYMBOL: + column.addSymbol(value); + break; + case ColumnType.BINARY: + column.addString(value); + break; + case ColumnType.CHAR: + column.addShort((short) stringToChar(value)); + break; + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + appendStringTimestamp(column, name, value, targetType); + break; + case ColumnType.UUID: + long lo; + long hi; + try { + checkUuid(value); + lo = parseUuidLo(value); + hi = parseUuidHi(value); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid UUID text"); + } + column.addUuid(hi, lo); + break; + case ColumnType.LONG256: + appendStringLong256(column, name, value); + break; + default: + throw unsupported(name, "STRING", targetType, "conversion is not implemented"); + } + } + return this; + } + + public QwpSchemaBinding charColumn(CharSequence name, char value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "CHAR", ColumnType.CHAR); + if (column != null) { + column.addShort((short) value); + } + return this; + } + + public QwpSchemaBinding symbol(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "SYMBOL"); + int targetType = targetType(index, ColumnType.SYMBOL); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "SYMBOL", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value); + break; + case ColumnType.SYMBOL: + column.addSymbol(value); + break; + default: + throw unsupported(name, "SYMBOL", targetType, "conversion is not implemented"); + } + return this; + } + + /** + * Appends a non-designated timestamp supplied in an explicitly supported fixed-duration unit + * to an exact TIMESTAMP, TIMESTAMP_NS, STRING or VARCHAR schema target. Text targets normalize + * through checked microseconds and use the server's UTC millisecond presentation. Conversion + * fails on overflow; nanos converted to micros truncate toward zero. The caller retains the + * class-level row-cancellation and uncommitted-column rollback responsibility on failure. + */ + public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUnit unit) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "TIMESTAMP"); + int targetType = targetType(index, + unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "TIMESTAMP", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "TIMESTAMP", targetType, "conversion is not implemented"); + } + if (unit == null) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp unit is null"); + } + if (!isSupportedTimestampUnit(unit)) { + throw unsupported(name, "TIMESTAMP", targetType, "timestamp unit is not supported: " + unit); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_NANO + ? NanosTimestampDriver.INSTANCE.from(value, unit) + : MicrosTimestampDriver.INSTANCE.from(value, unit); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + appendTimestampText(column, converted); + } else { + column.addLong(converted); + } + return this; + } + + /** + * Appends a non-designated Instant to an exact TIMESTAMP, TIMESTAMP_NS, STRING or VARCHAR + * schema target. Timestamp targets preserve their precision; text targets normalize through + * checked microseconds and use the server's UTC millisecond presentation. Final values outside + * the selected signed-long range are rejected. The caller retains the class-level rollback + * responsibility on failure. + */ + public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "TIMESTAMP"); + int targetType = targetType(index, ColumnType.TIMESTAMP_MICRO); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "TIMESTAMP", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "TIMESTAMP", targetType, "conversion is not implemented"); + } + if (value == null) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp value is null"); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_NANO + ? instantTimestamp(value, 1_000_000_000L, value.getNano()) + : instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + appendTimestampText(column, converted); + } else { + column.addLong(converted); + } + return this; + } + + public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "UUID"); + int targetType = targetType(index, ColumnType.UUID); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "UUID", index, targetType); + if (column == null) { + return this; + } + if (targetType == ColumnType.UUID) { + column.addUuid(hi, lo); + return this; + } + if (targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "UUID", targetType, "conversion is not implemented"); + } + if (lo == Long.MIN_VALUE && hi == Long.MIN_VALUE) { + column.addNull(); + } else { + column.addString(formatUuid(lo, hi)); + } + return this; + } + + /** Rejects an effective input family whose schema-mode conversion is not implemented. */ + public QwpSchemaBinding unsupportedColumn(CharSequence name, String inputType) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, inputType); + QwpTableBuffer.ColumnBuffer inferred = index < 0 ? buffer.getExistingInferredColumn(name) : null; + if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + return this; + } + if (index < 0) { + throw unsupported(name, inputType, -1, "input conversion is not implemented"); + } + int targetType = schema.getColumnType(index); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + if (column != null) { + throw unsupported(name, inputType, targetType, "input conversion is not implemented"); + } + return this; + } + + private static void checkUuid(CharSequence value) throws NumericException { + if (value.length() != UUID_LENGTH + || value.charAt(FIRST_DASH) != '-' + || value.charAt(SECOND_DASH) != '-' + || value.charAt(THIRD_DASH) != '-' + || value.charAt(FOURTH_DASH) != '-') { + throw NumericException.instance(); + } + } + + private static long parseUuidHi(CharSequence value) throws NumericException { + return (Numbers.parseHexLong(value, 0, FIRST_DASH) << 32) + | (Numbers.parseHexLong(value, FIRST_DASH + 1, SECOND_DASH) << 16) + | Numbers.parseHexLong(value, SECOND_DASH + 1, THIRD_DASH); + } + + private static long parseUuidLo(CharSequence value) throws NumericException { + return (Numbers.parseHexLong(value, THIRD_DASH + 1, FOURTH_DASH) << 48) + | Numbers.parseHexLong(value, FOURTH_DASH + 1, UUID_LENGTH); + } + + public int getTableId() { + return schema.getTableId(); + } + + public long getMetadataVersion() { + return schema.getMetadataVersion(); + } + + public boolean hasSameRelevantTarget(QwpSchemaResponse other, CharSequence name) { + int current = name == null ? schema.getDesignatedIndex() : columns.get(name); + int next = -1; + if (name == null) { + next = other.getDesignatedIndex(); + } else { + for (int i = 0, n = other.getColumnCount(); i < n; i++) { + if (Chars.equalsIgnoreCase(other.getColumnName(i), name)) { + next = i; + break; + } + } + } + if (current < 0 || next < 0) { + return current < 0 && next < 0; + } + return (current == schema.getDesignatedIndex()) == (next == other.getDesignatedIndex()) + && schema.getColumnType(current) == other.getColumnType(next) + && schema.hasColumnExtensionParameters(current) == other.hasColumnExtensionParameters(next); + } + + private LineSenderSchemaException error( + LineSenderSchemaException.Reason reason, + CharSequence column, + CharSequence inputType, + Integer targetType, + String detail + ) { + StringSink message = new StringSink(); + message.put("schema row error [reason=").put(reason.name()).put(", table=").putAsPrintable(tableName); + if (column != null) { + message.put(", column=").putAsPrintable(column); + } + if (inputType != null) { + message.put(", inputType=").put(inputType); + } + if (targetType != null) { + message.put(", targetType=").put(ColumnType.nameOf(targetType)).put('(').put(targetType).put(')'); + } + message.put(", detail=").put(detail).put(']'); + return new LineSenderSchemaException(reason, message); + } + + private void requireUsableSchema() { + switch (schema.getResult()) { + case QwpSchemaProtocol.RESULT_KNOWN: + case QwpSchemaProtocol.RESULT_MISSING: + return; + case QwpSchemaProtocol.RESULT_DENIED: + throw error(ACCESS_DENIED, null, null, null, "schema access denied"); + case QwpSchemaProtocol.RESULT_UNAVAILABLE: + throw error(SCHEMA_UNAVAILABLE, null, null, null, "schema is temporarily unavailable"); + case QwpSchemaProtocol.RESULT_TOO_LARGE: + throw error(UNSUPPORTED_FEATURE, null, null, null, "schema could not be returned within the discovery response limit"); + default: + throw error(UNSUPPORTED_FEATURE, null, null, null, "unknown schema result"); + } + } + + private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String inputType, int expectedTargetType) { + int index = targetIndex(name, inputType); + int targetType = targetType(index, expectedTargetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + if (column != null && targetType != expectedTargetType) { + throw unsupported(name, inputType, targetType, "conversion is not implemented"); + } + return column; + } + + private QwpTableBuffer.ColumnBuffer targetColumn( + CharSequence name, + String inputType, + int index, + int targetType + ) { + if (index < 0) { + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(name); + if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + return null; + } + if (inferred != null && inferred.getType() != wireType(targetType)) { + throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" + + QwpConstants.getTypeName(inferred.getType()) + ']'); + } + } + if (index >= 0 && index == schema.getDesignatedIndex()) { + throw unsupported(name, inputType, targetType, "designated timestamp writes are not implemented"); + } + final byte wireType = wireType(targetType); + if (wireType == 0) { + throw unsupported(name, inputType, targetType, "conversion is not implemented"); + } + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn(name, wireType, true); + if (column == null) { + return null; + } + if (index >= 0 && schema.hasColumnExtensionParameters(index)) { + throw unsupported(name, inputType, targetType, "parameterized target type"); + } + return column; + } + + private byte wireType(int targetType) { + if (ColumnType.isGeoHash(targetType)) { + int bits = ColumnType.getGeoHashBits(targetType); + return bits >= 1 && bits <= 60 && targetType == ColumnType.getGeoHashTypeWithBits(bits) + ? QwpConstants.TYPE_GEOHASH + : 0; + } + if (ColumnType.isDecimal(targetType)) { + int precision = ColumnType.getDecimalPrecision(targetType); + int scale = ColumnType.getDecimalScale(targetType); + if (precision < 1 || precision > Decimals.MAX_PRECISION || scale > precision + || targetType != ColumnType.getDecimalType(precision, scale)) { + return 0; + } + switch (ColumnType.tagOf(targetType)) { + case ColumnType.DECIMAL8: + case ColumnType.DECIMAL16: + case ColumnType.DECIMAL32: + case ColumnType.DECIMAL64: + return QwpConstants.TYPE_DECIMAL64; + case ColumnType.DECIMAL128: + return QwpConstants.TYPE_DECIMAL128; + case ColumnType.DECIMAL256: + return QwpConstants.TYPE_DECIMAL256; + default: + return 0; + } + } + switch (targetType) { + case ColumnType.BOOLEAN: + return QwpConstants.TYPE_BOOLEAN; + case ColumnType.BYTE: + return QwpConstants.TYPE_BYTE; + case ColumnType.SHORT: + return QwpConstants.TYPE_SHORT; + case ColumnType.CHAR: + return QwpConstants.TYPE_CHAR; + case ColumnType.INT: + return QwpConstants.TYPE_INT; + case ColumnType.LONG: + return QwpConstants.TYPE_LONG; + case ColumnType.FLOAT: + return QwpConstants.TYPE_FLOAT; + case ColumnType.DOUBLE: + return QwpConstants.TYPE_DOUBLE; + case ColumnType.STRING: + case ColumnType.VARCHAR: + return QwpConstants.TYPE_VARCHAR; + case ColumnType.SYMBOL: + return QwpConstants.TYPE_SYMBOL; + case ColumnType.UUID: + return QwpConstants.TYPE_UUID; + case ColumnType.LONG256: + return QwpConstants.TYPE_LONG256; + case ColumnType.BINARY: + return QwpConstants.TYPE_BINARY; + case ColumnType.TIMESTAMP_MICRO: + return QwpConstants.TYPE_TIMESTAMP; + case ColumnType.TIMESTAMP_NANO: + return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: + return 0; + } + } + + private void appendBinary( + CharSequence name, + QwpTableBuffer.ColumnBuffer column, + long ptr, + long len + ) { + if (len < 0) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary length must be non-negative"); + } + if (len > Integer.MAX_VALUE) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary length exceeds the supported maximum"); + } + if (len > 0 && ptr == 0) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary pointer cannot be zero for a non-empty value"); + } + column.addBinary(ptr, len); + } + + private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String inputType, byte inferredWireType) { + buffer.requireSchemaBinding(this); + if (value == null || value.isNull()) { + return this; + } + int index = targetIndex(name, inputType); + if (index < 0) { + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(name); + if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + return this; + } + if (inferred != null && inferred.getType() != inferredWireType) { + throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" + + QwpConstants.getTypeName(inferred.getType()) + ']'); + } + inferred = buffer.getOrCreateColumn(name, inferredWireType, true); + if (inferred == null) { + return this; + } + if (value instanceof Decimal64) { + inferred.addDecimal64((Decimal64) value); + } else if (value instanceof Decimal128) { + inferred.addDecimal128((Decimal128) value); + } else { + inferred.addDecimal256((Decimal256) value); + } + return this; + } + int targetType = targetType(index, ColumnType.DECIMAL256); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!ColumnType.isDecimal(targetType)) { + return decimalNonDecimalColumn(column, name, value, inputType, targetType); + } + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + try { + if (!column.addSchemaDecimal(value, targetPrecision, targetScale)) { + throw error(INVALID_VALUE, name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, inputType, targetType, "decimal value cannot be rescaled exactly"); + } + return this; + } + + private QwpSchemaBinding decimalNonDecimalColumn( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + Decimal value, + String inputType, + int targetType + ) { + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + Decimal256 decimal = decimalTextScratch; + if (decimal == null) { + decimalTextScratch = decimal = new Decimal256(); + } + value.toDecimal256(decimal); + decimal.setScale(decimal.getScale() & 0xff); + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(80); + } else { + sink.clear(); + } + decimal.toSink(sink); + column.addString(sink); + return this; + } + throw unsupported(name, inputType, targetType, "conversion is not implemented"); + } + + private LineSenderSchemaException invalidRange(CharSequence name, int targetType) { + return error(INVALID_VALUE, name, "LONG", targetType, "value is outside target type range"); + } + + private CharSequence formatLong(long value) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(20); + } else { + sink.clear(); + } + Numbers.append(sink, value, false); + return sink; + } + + private CharSequence formatFloating(double value) { + StringSink sink = floatingTextSink; + if (sink == null) { + floatingTextSink = sink = new StringSink(24); + floatingTextExponentScratch = new int[1]; + } else { + sink.clear(); + } + QwpSchemaDoubleFormatter.append(sink, value, floatingTextExponentScratch); + return sink; + } + + private CharSequence formatUuid(long lo, long hi) { + StringSink sink = uuidTextSink; + if (sink == null) { + uuidTextSink = sink = new StringSink(UUID_LENGTH); + } else { + sink.clear(); + } + appendUuidHex(sink, hi >>> 32, 8); + sink.put('-'); + appendUuidHex(sink, hi >>> 16, 4); + sink.put('-'); + appendUuidHex(sink, hi, 4); + sink.put('-'); + appendUuidHex(sink, lo >>> 48, 4); + sink.put('-'); + appendUuidHex(sink, lo, 12); + return sink; + } + + private static void appendUuidHex(StringSink sink, long value, int digits) { + for (int shift = (digits - 1) * 4; shift >= 0; shift -= 4) { + int nibble = (int) ((value >>> shift) & 0xf); + sink.put((char) (nibble < 10 ? '0' + nibble : 'a' + nibble - 10)); + } + } + + private LineSenderSchemaException invalidNumericText(CharSequence name, int targetType) { + return error(INVALID_VALUE, name, "STRING", targetType, "invalid numeric text"); + } + + private void appendStringNumeric( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + switch (targetType) { + case ColumnType.BYTE: + int parsed = Numbers.parseInt(value); + if (parsed < Byte.MIN_VALUE || parsed > Byte.MAX_VALUE) { + throw NumericException.instance(); + } + column.addByte((byte) parsed); + break; + case ColumnType.SHORT: + column.addShort(parseSchemaShort(value)); + break; + case ColumnType.INT: + column.addInt(Numbers.parseInt(value)); + break; + case ColumnType.LONG: + column.addLong(parseSchemaLong(value)); + break; + case ColumnType.FLOAT: + column.addFloat(parseSchemaFloat(value)); + break; + case ColumnType.DOUBLE: + column.addDouble(parseSchemaDouble(value)); + break; + default: + throw new AssertionError("unsupported numeric target"); + } + } catch (NumericException e) { + throw invalidNumericText(name, targetType); + } + } + + private void appendStringLong256( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value + ) { + int length = value.length(); + if (length < 4 || length > 66 || (length & 1) != 0 + || value.charAt(0) != '0' || value.charAt(1) != 'x') { + throw error(INVALID_VALUE, name, "STRING", (int) ColumnType.LONG256, "invalid LONG256 text"); + } + try { + int hi = length; + int lo = Math.max(2, hi - 16); + long l0 = Numbers.parseHexLong(value, lo, hi); + hi = lo; + lo = Math.max(2, hi - 16); + long l1 = hi > 2 ? Numbers.parseHexLong(value, lo, hi) : 0; + hi = lo; + lo = Math.max(2, hi - 16); + long l2 = hi > 2 ? Numbers.parseHexLong(value, lo, hi) : 0; + hi = lo; + long l3 = hi > 2 ? Numbers.parseHexLong(value, 2, hi) : 0; + if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE + && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { + throw NumericException.instance(); + } + column.addLong256(l0, l1, l2, l3); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", (int) ColumnType.LONG256, "invalid LONG256 text"); + } + } + + private void appendStringGeoHash( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int precision, + int targetType + ) { + int chars = Math.min(value.length(), 12); + if (chars * 5 < precision) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid GEOHASH text"); + } + try { + long hash = Numbers.parseGeoHashBase32(value, 0, chars); + column.addGeoHash(hash >>> (chars * 5 - precision), precision); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid GEOHASH text"); + } + } + + private void appendStringTimestamp( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + Utf8StringSink sink = timestampTextSink; + if (sink == null) { + timestampTextSink = sink = new Utf8StringSink(Math.max(32, value.length())); + } else { + sink.clear(); + } + sink.put(value); + try { + long micros = QwpSchemaTimestampParser.parse(sink.asAsciiCharSequence()); + if (targetType == ColumnType.TIMESTAMP_NANO) { + if (micros > Long.MAX_VALUE / 1000 || micros < Long.MIN_VALUE / 1000) { + throw NumericException.instance(); + } + column.addLong(micros * 1000); + } else { + column.addLong(micros); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid timestamp text"); + } + } + + private void appendTimestampText(QwpTableBuffer.ColumnBuffer column, long micros) { + Utf8StringSink sink = timestampTextSink; + if (sink == null) { + timestampTextSink = sink = new Utf8StringSink(32); + } else { + sink.clear(); + } + QwpSchemaTimestampFormatter.appendDateTime(sink, micros); + column.addString(sink.asAsciiCharSequence()); + } + + private static double parseSchemaDouble(CharSequence value) throws NumericException { + int length = oppositeFloatingSuffixLength(value, 'f'); + return length < 0 + ? Numbers.parseDouble(value) + : FastDoubleParser.parseDouble(value, 0, length, true); + } + + private static float parseSchemaFloat(CharSequence value) throws NumericException { + int length = oppositeFloatingSuffixLength(value, 'd'); + return length < 0 + ? FastFloatParser.parseFloat(value, true) + : FastFloatParser.parseFloat(value, 0, length, true); + } + + private static int oppositeFloatingSuffixLength(CharSequence value, char suffix) { + int index = value.length() - 1; + while (index >= 0 && value.charAt(index) <= ' ') { + index--; + } + if (index <= 0 || (value.charAt(index) | 32) != suffix) { + return -1; + } + char preceding = value.charAt(index - 1); + return (preceding >= '0' && preceding <= '9') || preceding == '.' ? index : -1; + } + + private static long parseSchemaLong(CharSequence value) throws NumericException { + int length = value.length(); + if (length == 0) { + throw NumericException.instance(); + } + boolean negative = value.charAt(0) == '-'; + int index = negative ? 1 : 0; + if (index == length) { + throw NumericException.instance(); + } + long limit = negative ? Long.MIN_VALUE : -Long.MAX_VALUE; + long multiplyLimit = limit / 10; + long result = 0; + int digitCount = 0; + for (; index < length; index++) { + char c = value.charAt(index); + if (c == '_' || c == 0x7f) { + if (digitCount == 0) { + throw NumericException.instance(); + } + digitCount = 0; + } else if (c == 'L' || c == 'l') { + if (digitCount == 0 || index + 1 != length) { + throw NumericException.instance(); + } + } else { + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + int digit = c - '0'; + if (result < multiplyLimit) { + throw NumericException.instance(); + } + result *= 10; + if (result < limit + digit) { + throw NumericException.instance(); + } + result -= digit; + digitCount++; + } + } + if (digitCount == 0) { + throw NumericException.instance(); + } + return negative ? result : -result; + } + + private static short parseSchemaShort(CharSequence value) throws NumericException { + int length = value.length(); + int index = length > 0 && value.charAt(0) == '-' ? 1 : 0; + if (index == length) { + throw NumericException.instance(); + } + for (; index < length; index++) { + char c = value.charAt(index); + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + } + int parsed = Numbers.parseInt(value); + if (parsed < Short.MIN_VALUE || parsed > Short.MAX_VALUE) { + throw NumericException.instance(); + } + return (short) parsed; + } + + private QwpSchemaBinding floatingNumericColumn(CharSequence name, double value, String inputType) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, inputType); + int targetType = targetType(index, + "FLOAT".equals(inputType) ? ColumnType.FLOAT : ColumnType.DOUBLE); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType)) { + return floatingNonNumericColumn(column, name, value, inputType, targetType); + } + if (Double.isNaN(value)) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + requireWholeInRange(name, inputType, targetType, value, Byte.MIN_VALUE, Byte.MAX_VALUE); + column.addByte((byte) value); + break; + case ColumnType.SHORT: + requireWholeInRange(name, inputType, targetType, value, Short.MIN_VALUE, Short.MAX_VALUE); + column.addShort((short) value); + break; + case ColumnType.INT: + requireWholeInRange(name, inputType, targetType, value, Integer.MIN_VALUE, Integer.MAX_VALUE); + column.addInt((int) value); + break; + case ColumnType.LONG: + if (!Double.isFinite(value) || value != Math.rint(value) || value < -0x1.0p63 || value >= 0x1.0p63) { + throw error(INVALID_VALUE, name, inputType, targetType, "value cannot be represented exactly by target integer type"); + } + column.addLong((long) value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble(value); + break; + } + return this; + } + + private QwpSchemaBinding floatingNonNumericColumn( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + double value, + String inputType, + int targetType + ) { + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (!Double.isFinite(value)) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaStringDecimal(formatFloating(value), targetPrecision, targetScale)) { + throw error(INVALID_VALUE, name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, inputType, targetType, + "floating value cannot be represented exactly by target decimal precision and scale"); + } + return this; + } + if (!isTextTarget(targetType)) { + throw unsupported(name, inputType, targetType, "conversion is not implemented"); + } + if (Double.isNaN(value)) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatFloating(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatFloating(value)); + break; + default: + throw unsupported(name, inputType, targetType, "conversion is not implemented"); + } + return this; + } + + private void requireWholeInRange( + CharSequence name, + String inputType, + int targetType, + double value, + double min, + double max + ) { + if (!Double.isFinite(value) || value != Math.rint(value) || value < min || value > max) { + throw error(INVALID_VALUE, name, inputType, targetType, "value cannot be represented exactly by target integer type"); + } + } + + private static boolean isNumericTarget(int targetType) { + return targetType == ColumnType.BYTE + || targetType == ColumnType.SHORT + || targetType == ColumnType.INT + || targetType == ColumnType.LONG + || targetType == ColumnType.FLOAT + || targetType == ColumnType.DOUBLE; + } + + private static boolean isTextTarget(int targetType) { + return targetType == ColumnType.STRING + || targetType == ColumnType.VARCHAR + || targetType == ColumnType.SYMBOL; + } + + private static boolean isSupportedTimestampUnit(ChronoUnit unit) { + return unit == ChronoUnit.NANOS + || unit == ChronoUnit.MICROS + || unit == ChronoUnit.MILLIS + || unit == ChronoUnit.SECONDS + || unit == ChronoUnit.MINUTES + || unit == ChronoUnit.HOURS + || unit == ChronoUnit.DAYS; + } + + // Matches client UTF-8 replacement followed by the server's first-BMP decoder: + // supplementary code points become NUL and malformed surrogates become '?'. + private static char stringToChar(CharSequence value) { + if (value.length() == 0) { + return 0; + } + char first = value.charAt(0); + if (Character.isHighSurrogate(first)) { + return value.length() > 1 && Character.isLowSurrogate(value.charAt(1)) ? 0 : '?'; + } + return Character.isLowSurrogate(first) ? '?' : first; + } + + private static long instantTimestamp(Instant instant, long unitsPerSecond, long fraction) { + long seconds = instant.getEpochSecond(); + if (seconds < 0) { + // Keep both terms non-positive so an intermediate underflow also means the final sum is out of range. + return Math.addExact(Math.multiplyExact(seconds + 1, unitsPerSecond), fraction - unitsPerSecond); + } + return Math.addExact(Math.multiplyExact(seconds, unitsPerSecond), fraction); + } + + private int requireDesignatedTimestamp(String inputType) { + int index = schema.getDesignatedIndex(); + if (index < 0) { + throw unsupported(null, inputType, -1, "target table has no designated timestamp"); + } + int targetType = schema.getColumnType(index); + timestampWireType(targetType); + if (schema.hasColumnExtensionParameters(index)) { + throw unsupported(null, inputType, targetType, "parameterized target type"); + } + return index; + } + + private byte timestampWireType(int targetType) { + if (targetType == ColumnType.TIMESTAMP_MICRO) { + return QwpConstants.TYPE_TIMESTAMP; + } + if (targetType == ColumnType.TIMESTAMP_NANO) { + return QwpConstants.TYPE_TIMESTAMP_NANOS; + } + throw unsupported(null, "TIMESTAMP", targetType, "designated timestamp conversion is not implemented"); + } + + private int targetIndex(CharSequence name, String inputType) { + if (name == null || !TableUtils.isValidColumnName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { + throw error(INVALID_VALUE, name, inputType, null, "invalid column name"); + } + int index = columns.get(name); + return index; + } + + private int targetType(int index, int inferredType) { + return index >= 0 ? schema.getColumnType(index) : inferredType; + } + + private LineSenderSchemaException unsupported(CharSequence name, String inputType, int targetType, String detail) { + return error(UNSUPPORTED_FEATURE, name, inputType, targetType, detail); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java new file mode 100644 index 000000000..30557a26a --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java @@ -0,0 +1,103 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.Numbers; +import io.questdb.client.std.str.StringSink; + +/** Schema-only floating formatter matching the server's Ryu presentation. */ +final class QwpSchemaDoubleFormatter { + private static final int EXP_SHIFT = 52; + private static final long EXP_BIT_MASK = 0x7ff0000000000000L; + private static final long SIGN_BIT_MASK = 0x8000000000000000L; + private static final long SIGNIFICAND_BIT_MASK = 0x000fffffffffffffL; + + private QwpSchemaDoubleFormatter() { + } + + static void append(StringSink sink, double value, int[] exponentScratch) { + long bits = Double.doubleToRawLongBits(value); + long mantissa = bits & SIGNIFICAND_BIT_MASK; + int exponent = (int) ((bits & EXP_BIT_MASK) >>> EXP_SHIFT); + if (exponent == 2047) { + if (mantissa != 0) { + sink.putAscii("NaN"); + } else if ((bits & SIGN_BIT_MASK) != 0) { + sink.putAscii("-Infinity"); + } else { + sink.putAscii("Infinity"); + } + return; + } + if ((bits & SIGN_BIT_MASK) != 0) { + sink.putAscii('-'); + } + if (exponent == 0 && mantissa == 0) { + sink.putAscii("0.0"); + return; + } + long output = QwpSchemaRyuDouble.d2d(mantissa, exponent, exponentScratch); + int length = QwpSchemaRyuDouble.decimalLength17(output); + int decimalExponent = exponentScratch[0] + length; + if (decimalExponent > 0 && decimalExponent < 8) { + appendFixedPositive(sink, output, length, decimalExponent); + } else if (decimalExponent <= 0 && decimalExponent > -3) { + appendFixedLeadingZero(sink, output, length, decimalExponent); + } else { + appendScientific(sink, output, length, decimalExponent); + } + } + + private static void appendFixedLeadingZero( + StringSink sink, long output, int length, int decimalExponent + ) { + sink.putAscii("0."); + for (int i = 0; i < -decimalExponent; i++) { + sink.putAscii('0'); + } + appendDigits(sink, output, length, 0, length); + } + + private static void appendFixedPositive( + StringSink sink, long output, int length, int decimalExponent + ) { + if (length <= decimalExponent) { + appendDigits(sink, output, length, 0, length); + for (int i = length; i < decimalExponent; i++) { + sink.putAscii('0'); + } + sink.putAscii(".0"); + } else { + appendDigits(sink, output, length, 0, decimalExponent); + sink.putAscii('.'); + appendDigits(sink, output, length, decimalExponent, length); + } + } + + private static void appendScientific( + StringSink sink, long output, int length, int decimalExponent + ) { + appendDigit(sink, output, length - 1); + sink.putAscii('.'); + if (length > 1) { + appendDigits(sink, output, length, 1, length); + } else { + sink.putAscii('0'); + } + sink.putAscii('E'); + Numbers.append(sink, decimalExponent - 1); + } + + private static void appendDigits(StringSink sink, long output, int length, int from, int to) { + for (int i = from; i < to; i++) { + appendDigit(sink, output, length - 1 - i); + } + } + + private static void appendDigit(StringSink sink, long output, int power) { + sink.putAscii((char) ('0' + (int) (output / Numbers.pow10[power] % 10))); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java new file mode 100644 index 000000000..4b81efbcc --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java @@ -0,0 +1,285 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; +import io.questdb.client.std.Unsafe; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Codec and wire constants for negotiated QWP schema control messages. */ +public final class QwpSchemaProtocol { + public static final byte FLAG_CONTROL = 0x20; + public static final byte KIND_DESCRIBE = 1; + public static final byte KIND_SCHEMA = 2; + public static final int MAX_COLUMN_COUNT = 2048; + public static final int MAX_FRAME_SIZE = 1024 * 1024; + public static final int MAX_NAME_UTF16_LENGTH = 127; + public static final int MAX_NAME_UTF8_LENGTH = 381; + public static final int MAX_PARAMS_LENGTH = 1024; + public static final int RESULT_DENIED = 2; + public static final int RESULT_KNOWN = 0; + public static final int RESULT_MISSING = 1; + public static final int RESULT_TOO_LARGE = 4; + public static final int RESULT_UNAVAILABLE = 3; + + private static final int KNOWN_FIXED_PAYLOAD_SIZE = 1 + 8 + 1 + 4 + 8 + 2 + 2; + private static final int MIN_PAYLOAD_SIZE = 1 + 8 + 1; + + private QwpSchemaProtocol() { + } + + public static QwpSchemaResponse decodeResponse(long address, int length) { + int payloadLength = validateResponseEnvelope(address, length, MIN_PAYLOAD_SIZE); + return decodePayload(address + QwpConstants.HEADER_SIZE, payloadLength, false); + } + + /** Validates a complete schema-response envelope and returns its positive request ID. */ + public static long peekResponseRequestId(long address, int length) { + validateResponseEnvelope(address, length, 1 + Long.BYTES); + long payloadAddress = address + QwpConstants.HEADER_SIZE; + if (getUnsignedByte(payloadAddress) != KIND_SCHEMA) { + throw malformed("unexpected control response kind"); + } + long requestId = getLong(payloadAddress + 1); + if (requestId <= 0) { + throw malformed("request id must be positive"); + } + return requestId; + } + + private static int validateResponseEnvelope(long address, int length, int minimumPayloadSize) { + if (address == 0) { + throw malformed("null frame address"); + } + if (length < QwpConstants.HEADER_SIZE + minimumPayloadSize || length > MAX_FRAME_SIZE) { + throw malformed("invalid frame length: " + length); + } + if (getInt(address) != QwpConstants.MAGIC_MESSAGE) { + throw malformed("invalid QWP magic"); + } + if (getUnsignedByte(address + 4) != 1) { + throw malformed("unsupported QWP version"); + } + if (getUnsignedByte(address + 5) != (FLAG_CONTROL & 0xff)) { + throw malformed("invalid control flags"); + } + if (getUnsignedShort(address + 6) != 0) { + throw malformed("control frame must have zero tables"); + } + int payloadLength = getInt(address + 8); + if (payloadLength != length - QwpConstants.HEADER_SIZE) { + throw malformed("payload length mismatch"); + } + return payloadLength; + } + + /** Decodes a length-delimited schema payload carried by response feedback. */ + public static QwpSchemaResponse decodeFeedbackPayload(long address, int length) { + if (address == 0) { + throw malformed("null payload address"); + } + if (length < MIN_PAYLOAD_SIZE || length > MAX_FRAME_SIZE) { + throw malformed("invalid payload length: " + length); + } + return decodePayload(address, length, true); + } + + private static QwpSchemaResponse decodePayload(long address, int length, boolean feedback) { + long p = address; + long limit = address + length; + if (getUnsignedByte(p++) != KIND_SCHEMA) { + throw malformed("unexpected control response kind"); + } + long requestId = getLong(p); + p += 8; + if (feedback ? requestId != 0 : requestId <= 0) { + throw malformed(feedback ? "feedback request id must be zero" : "request id must be positive"); + } + int result = getUnsignedByte(p++); + if (result < RESULT_KNOWN || result > RESULT_TOO_LARGE) { + throw malformed("unknown schema result: " + result); + } + if (result != RESULT_KNOWN) { + if (p != limit) { + throw malformed("trailing bytes after schema result"); + } + return new QwpSchemaResponse(requestId, result, -1, -1, -1, new QwpSchemaResponse.Column[0]); + } + if (length < KNOWN_FIXED_PAYLOAD_SIZE) { + throw malformed("truncated known schema"); + } + require(p, 16, limit, "schema identity"); + int tableId = getInt(p); + p += 4; + long metadataVersion = getLong(p); + p += 8; + int designatedIndex = getShort(p); + p += 2; + int columnCount = getUnsignedShort(p); + p += 2; + if (tableId < 0 || metadataVersion < 0) { + throw malformed("negative schema identity"); + } + if (columnCount > MAX_COLUMN_COUNT) { + throw malformed("too many columns: " + columnCount); + } + if (designatedIndex < -1 || designatedIndex >= columnCount) { + throw malformed("invalid designated index: " + designatedIndex); + } + + QwpSchemaResponse.Column[] columns = new QwpSchemaResponse.Column[columnCount]; + LowerCaseCharSequenceIntHashMap columnNames = new LowerCaseCharSequenceIntHashMap(columnCount); + for (int i = 0; i < columnCount; i++) { + require(p, 2, limit, "column name length"); + int nameLength = getUnsignedShort(p); + p += 2; + if (nameLength == 0 || nameLength > MAX_NAME_UTF8_LENGTH) { + throw malformed("invalid column name length: " + nameLength); + } + require(p, nameLength, limit, "column name"); + String name = decodeUtf8(p, nameLength, "column name"); + if (!TableUtils.isValidColumnName(name, MAX_NAME_UTF16_LENGTH)) { + throw malformed("invalid column name"); + } + if (!columnNames.put(name, i)) { + throw malformed("duplicate case-insensitive column name"); + } + p += nameLength; + require(p, 6, limit, "column type and parameter length"); + int type = getInt(p); + p += 4; + int paramsLength = getUnsignedShort(p); + p += 2; + if (paramsLength > MAX_PARAMS_LENGTH) { + throw malformed("column parameters too large: " + paramsLength); + } + require(p, paramsLength, limit, "column parameters"); + p += paramsLength; + columns[i] = new QwpSchemaResponse.Column(name, type, paramsLength != 0); + } + if (p != limit) { + throw malformed("trailing bytes after schema"); + } + return new QwpSchemaResponse(requestId, result, tableId, metadataVersion, designatedIndex, columns); + } + + public static byte[] encodeDescribe(long requestId, CharSequence tableName) { + if (requestId <= 0) { + throw new IllegalArgumentException("request id must be positive"); + } + if (tableName == null || !TableUtils.isValidTableName(tableName, MAX_NAME_UTF16_LENGTH)) { + throw new IllegalArgumentException("invalid table name"); + } + byte[] name = encodeName(tableName, "table name"); + int payloadLength = 1 + 8 + 2 + name.length; + ByteBuffer buffer = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(QwpConstants.MAGIC_MESSAGE); + buffer.put((byte) 1); + buffer.put(FLAG_CONTROL); + buffer.putShort((short) 0); + buffer.putInt(payloadLength); + buffer.put(KIND_DESCRIBE); + buffer.putLong(requestId); + buffer.putShort((short) name.length); + buffer.put(name); + return buffer.array(); + } + + private static byte[] copyBytes(long address, int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + return bytes; + } + + private static String decodeUtf8(long address, int length, String what) { + byte[] bytes = copyBytes(address, length); + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw malformed("invalid UTF-8 " + what); + } + } + + private static byte[] encodeName(CharSequence value, String what) { + if (value == null || value.length() == 0 || value.length() > MAX_NAME_UTF16_LENGTH) { + throw new IllegalArgumentException(what + " must contain 1 to " + MAX_NAME_UTF16_LENGTH + " UTF-16 units"); + } + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (Character.isHighSurrogate(c)) { + if (++i >= value.length() || !Character.isLowSurrogate(value.charAt(i))) { + throw new IllegalArgumentException(what + " contains an unpaired surrogate"); + } + } else if (Character.isLowSurrogate(c)) { + throw new IllegalArgumentException(what + " contains an unpaired surrogate"); + } + } + byte[] bytes = value.toString().getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_NAME_UTF8_LENGTH) { + throw new IllegalArgumentException(what + " exceeds " + MAX_NAME_UTF8_LENGTH + " UTF-8 bytes"); + } + return bytes; + } + + private static int getInt(long address) { + return Unsafe.getUnsafe().getInt(address); + } + + private static long getLong(long address) { + return Unsafe.getUnsafe().getLong(address); + } + + private static short getShort(long address) { + return Unsafe.getUnsafe().getShort(address); + } + + private static int getUnsignedByte(long address) { + return Unsafe.getUnsafe().getByte(address) & 0xff; + } + + private static int getUnsignedShort(long address) { + return getShort(address) & 0xffff; + } + + private static IllegalArgumentException malformed(String message) { + return new IllegalArgumentException("Malformed QWP schema response: " + message); + } + + private static void require(long position, int needed, long limit, String what) { + if (needed < 0 || position > limit || needed > limit - position) { + throw malformed("truncated " + what); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java new file mode 100644 index 000000000..9cbaf2efc --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java @@ -0,0 +1,101 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +/** Immutable response to a QWP schema DESCRIBE control request. */ +public final class QwpSchemaResponse { + private final Column[] columns; + private final int designatedIndex; + private final int result; + private final int tableId; + private final long metadataVersion; + private final long requestId; + + QwpSchemaResponse(long requestId, int result, int tableId, long metadataVersion, int designatedIndex, Column[] columns) { + this.requestId = requestId; + this.result = result; + this.tableId = tableId; + this.metadataVersion = metadataVersion; + this.designatedIndex = designatedIndex; + this.columns = columns; + } + + public int getColumnCount() { + return columns.length; + } + + public String getColumnName(int index) { + return columns[index].name; + } + + public int getColumnType(int index) { + return columns[index].type; + } + + /** + * Returns whether the server supplied an extension parameter block for + * this column's type. The current client safely skips those bytes but must + * not interpret a parameterized type as an unparameterized one. + */ + public boolean hasColumnExtensionParameters(int index) { + return columns[index].hasTypeParameters; + } + + public int getDesignatedIndex() { + return designatedIndex; + } + + public long getMetadataVersion() { + return metadataVersion; + } + + /** Returns the positive DESCRIBE id, or zero for named write feedback. */ + public long getRequestId() { + return requestId; + } + + public int getResult() { + return result; + } + + public int getTableId() { + return tableId; + } + + public boolean hasSchema() { + return result == QwpSchemaProtocol.RESULT_KNOWN; + } + + static final class Column { + private final boolean hasTypeParameters; + private final String name; + private final int type; + + Column(String name, int type, boolean hasTypeParameters) { + this.name = name; + this.type = type; + this.hasTypeParameters = hasTypeParameters; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java new file mode 100644 index 000000000..edba00986 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java @@ -0,0 +1,635 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +/** + * Schema-only port of the QuestDB server's {@code io.questdb.std.RyuDouble} + * at revision {@code 12a33d651e51e2682e7a448c8db5168fc72dfad3}. + * Ryu algorithm for converting IEEE 754 double-precision floating-point numbers + * to their shortest decimal representation. + *

+ * Based on the paper "Ryu: Fast Float-to-String Conversion" by Ulf Adams (2018). + * This implementation uses 128-bit arithmetic via {@code Math.multiplyHigh} (Java 9+). + *

+ * The core method {@link #d2d} returns the shortest decimal significand as a {@code long} + * (1-17 digits) and writes the decimal exponent to a caller-provided array. The result + * satisfies: {@code value = output * 10^e10}. + */ +class QwpSchemaRyuDouble { + private static final int DOUBLE_POW5_BITCOUNT = 125; + private static final int DOUBLE_POW5_INV_BITCOUNT = 125; + // ceil(2^(pow5bits(q) - 1 + 125) / 5^q) for q in [0, 291], stored as (hi64, lo64) pairs. + private static final long[] DOUBLE_POW5_INV_SPLIT = initPow5InvSplit(); + // Upper 128 bits of 5^i for i in [0, 325], stored as (hi64, lo64) pairs. + private static final long[] DOUBLE_POW5_SPLIT = initPow5Split(); + + private static long[] initPow5InvSplit() { + return new long[]{ + 0x2000000000000000L, 0x0000000000000001L, 0x1999999999999999L, 0x999999999999999AL, + 0x147AE147AE147AE1L, 0x47AE147AE147AE15L, 0x10624DD2F1A9FBE7L, 0x6C8B4395810624DEL, + 0x1A36E2EB1C432CA5L, 0x7A786C226809D496L, 0x14F8B588E368F084L, 0x61F9F01B866E43ABL, + 0x10C6F7A0B5ED8D36L, 0xB4C7F34938583622L, 0x1AD7F29ABCAF4857L, 0x87A6520EC08D236AL, + 0x15798EE2308C39DFL, 0x9FB841A566D74F88L, 0x112E0BE826D694B2L, 0xE62D01511F12A607L, + 0x1B7CDFD9D7BDBAB7L, 0xD6AE6881CB5109A4L, 0x15FD7FE17964955FL, 0xDEF1ED34A2A73AEAL, + 0x119799812DEA1119L, 0x7F27F0F6E885C8BBL, 0x1C25C268497681C2L, 0x650CB4BE40D60DF8L, + 0x16849B86A12B9B01L, 0xEA70909833DE7193L, 0x1203AF9EE756159BL, 0x21F3A6E0297EC143L, + 0x1CD2B297D889BC2BL, 0x6985D7CD0F313537L, 0x170EF54646D49689L, 0x2137DFD73F5A90F9L, + 0x12725DD1D243ABA0L, 0xE75FE645CC4873FAL, 0x1D83C94FB6D2AC34L, 0xA5663D3C7A0D865DL, + 0x179CA10C9242235DL, 0x511E976394D79EB1L, 0x12E3B40A0E9B4F7DL, 0xDA7EDF82DD794BC1L, + 0x1E392010175EE596L, 0x2A6498D1625BAC68L, 0x182DB34012B25144L, 0xEEB6E0A781E2F053L, + 0x1357C299A88EA76AL, 0x58924D52CE4F26A9L, 0x1EF2D0F5DA7DD8AAL, 0x27507BB7B07EA441L, + 0x18C240C4AECB13BBL, 0x52A6C95FC0655034L, 0x13CE9A36F23C0FC9L, 0x0EEBD44C99EAA690L, + 0x1FB0F6BE50601941L, 0xB17953ADC3110A80L, 0x195A5EFEA6B34767L, 0xC12DDC8B02740867L, + 0x14484BFEEBC29F86L, 0x3424B06F3529A052L, 0x1039D66589687F9EL, 0x901D59F290EE19DBL, + 0x19F623D5A8A73297L, 0x4CFBC31DB4B0295FL, 0x14C4E977BA1F5BACL, 0x3D9635B15D59BAB2L, + 0x109D8792FB4C4956L, 0x97AB5E277DE16228L, 0x1A95A5B7F87A0EF0L, 0xF2ABC9D8C9689D0DL, + 0x154484932D2E725AL, 0x5BBCA17A3ABA173EL, 0x11039D428A8B8EAEL, 0xAFCA1AC82EFB45CBL, + 0x1B38FB9DAA78E44AL, 0xB2DCF7A6B1920945L, 0x15C72FB1552D836EL, 0xF57D92EBC141A104L, + 0x116C262777579C58L, 0xC46475896767B403L, 0x1BE03D0BF225C6F4L, 0x6D6D88DBD8A5ECD2L, + 0x164CFDA3281E38C3L, 0x8ABE071646EB23DBL, 0x11D7314F534B609CL, 0x6EFE6C11D255B649L, + 0x1C8B821885456760L, 0xB197134FB6EF8A0EL, 0x16D601AD376AB91AL, 0x27AC0F72F8BFA1A5L, + 0x1244CE242C5560E1L, 0xB95672C260994E1EL, 0x1D3AE36D13BBCE35L, 0xF5571E03CDC21695L, + 0x17624F8A762FD82BL, 0x2AAC18030B01ABABL, 0x12B50C6EC4F31355L, 0xBBBCE0026F348956L, + 0x1DEE7A4AD4B81EEFL, 0x92C7CCD0B1EDA889L, 0x17F1FB6F10934BF2L, 0xDBD30A408E57BA07L, + 0x1327FC58DA0F6FF5L, 0x7CA8D50071DFC806L, 0x1EA6608E29B24CBBL, 0xFAA7BB33E9660CD6L, + 0x18851A0B548EA3C9L, 0x9552FC298784D711L, 0x139DAE6F76D88307L, 0xAAA8C9BAD2D0AC0EL, + 0x1F62B0B257C0D1A5L, 0xDDDADC5E1E1AACE3L, 0x191BC08EAC9A4151L, 0x7E48B04B4B488A4FL, + 0x141633A556E1CDDAL, 0xCB6D59D5D5D3A1D9L, 0x1011C2EAABE7D7E2L, 0x3C577B1177DC817BL, + 0x19B604AAACA62636L, 0xC6F25E825960CF2AL, 0x14919D5556EB51C5L, 0x6BF518684780A5BBL, + 0x10747DDDDF22A7D1L, 0x232A79ED06008496L, 0x1A53FC9631D10C81L, 0xD1DD8FE1A3340756L, + 0x150FFD44F4A73D34L, 0xA7E4731AE8F66C45L, 0x10D9976A5D52975DL, 0x531D28E253F8569EL, + 0x1AF5BF109550F22EL, 0xEB61DB03B98D5762L, 0x159165A6DDDA5B58L, 0xBC4E48CFC7A445E8L, + 0x11411E1F17E1E2ADL, 0x6371D3D96C836B20L, 0x1B9B6364F3030448L, 0x9F1C8628AD9F11CDL, + 0x1615E91D8F359D06L, 0xE5B06B53BE18DB0BL, 0x11AB20E472914A6BL, 0xEAF3890FCB4715A2L, + 0x1C45016D841BAA46L, 0x44B8DB4C7871BC37L, 0x169D9ABE03495505L, 0x03C715D6C6C1635FL, + 0x1217AEFE69077737L, 0x3638DE456BCDE919L, 0x1CF2B1970E725858L, 0x56C163A2461641C1L, + 0x17288E1271F51379L, 0xDF011C81D1AB67CEL, 0x1286D80EC190DC61L, 0x7F3416CE4155ECA5L, + 0x1DA48CE468E7C702L, 0x6520247D3556476EL, 0x17B6D71D20B96C01L, 0xEA801D30F7783925L, + 0x12F8AC174D612334L, 0xBB99B0F3F92CFA84L, 0x1E5AACF215683854L, 0x5F5C4E532847F739L, + 0x18488A5B44536043L, 0x7F7D0B75B9D32C2EL, 0x136D3B7C36A919CFL, 0x9930D5F7C7DC2358L, + 0x1F152BF9F10E8FB2L, 0x8EB4898C72F9D226L, 0x18DDBCC7F40BA628L, 0x722A07A38F2E41B8L, + 0x13E497065CD61E86L, 0xC1BB394FA5BE9AFAL, 0x1FD424D6FAF030D7L, 0x9C5EC2190930F7F6L, + 0x197683DF2F268D79L, 0x49E56814075A5FF8L, 0x145ECFE5BF520AC7L, 0x6E51201005E1E660L, + 0x104BD984990E6F05L, 0xF1DA800CD181851AL, 0x1A12F5A0F4E3E4D6L, 0x4FC400148268D4F5L, + 0x14DBF7B3F71CB711L, 0xD96999AA01ED772BL, 0x10AFF95CC5B09274L, 0xADEE1488018AC5BCL, + 0x1AB328946F80EA54L, 0x497CEDA668DE092CL, 0x155C2076BF9A5510L, 0x3ACA57B853E4D424L, + 0x1116805EFFAEAA73L, 0x623B7960431D7683L, 0x1B5733CB32B110B8L, 0x9D2BF566D1C8BD9EL, + 0x15DF5CA28EF40D60L, 0x7DBCC452416D647FL, 0x117F7D4ED8C33DE6L, 0xCAFD69DB678AB6CCL, + 0x1BFF2EE48E052FD7L, 0xAB2F0FC572778ADFL, 0x1665BF1D3E6A8CACL, 0x88F273045B92D580L, + 0x11EAFF4A98553D56L, 0xD3F528D049424466L, 0x1CAB3210F3BB9557L, 0xB988414D4203A0A3L, + 0x16EF5B40C2FC7779L, 0x6139CDD76802E6E9L, 0x125915CD68C9F92DL, 0xE761717920025254L, + 0x1D5B561574765B7CL, 0xA568B58E999D5086L, 0x177C44DDF6C515FDL, 0x5120913EE14AA6D2L, + 0x12C9D0B1923744CAL, 0xA74D40FF1AA21F0EL, 0x1E0FB44F50586E11L, 0x0BAECE64F769CB4AL, + 0x180C903F7379F1A7L, 0x3C8BD850C5EE3C3BL, 0x133D4032C2C7F485L, 0xCA0979DA37F1C9C9L, + 0x1EC866B79E0CBA6FL, 0xA9A8C2F6BFE942DBL, 0x18A0522C7E709526L, 0x2153CF2BCCBA9BE3L, + 0x13B374F06526DDB8L, 0x1AA9728970954982L, 0x1F8587E7083E2F8CL, 0xF775840F1A88759DL, + 0x19379FEC0698260AL, 0x5F9136727BA05E17L, 0x142C7FF0054684D5L, 0x1940F85B9619E4DFL, + 0x1023998CD1053710L, 0xE100C6AFAB47EA4CL, 0x19D28F47B4D524E7L, 0xCE67A44C453FDD47L, + 0x14A8729FC3DDB71FL, 0xD852E9D69DCCB106L, 0x1086C219697E2C19L, 0x79DBEE454B0A2738L, + 0x1A71368F0F30468FL, 0x295FE3A211A9D859L, 0x15275ED8D8F36BA5L, 0xBAB31C81A7BB137AL, + 0x10EC4BE0AD8F8951L, 0x6228E39AEC95A92FL, 0x1B13AC9AAF4C0EE8L, 0x9D0E38F7E0EF7517L, + 0x15A956E225D67253L, 0xB0D82D931A592A79L, 0x11544581B7DEC1DCL, 0x8D79BE0F4847552EL, + 0x1BBA08CF8C979C94L, 0x158F967EDA0BBB7CL, 0x162E6D72D6DFB076L, 0x77A611FF14D62F97L, + 0x11BEBDF578B2F391L, 0xF951A7FF43DE8C79L, 0x1C6463225AB7EC1CL, 0xC21C3FFED2FDAD8EL, + 0x16B6B5B5155FF017L, 0x01B0333242648AD8L, 0x122BC490DDE659ACL, 0x0159C28E9B83A246L, + 0x1D12D41AFCA3C2ACL, 0xCEF604175F3903A3L, 0x17424348CA1C9BBDL, 0x725E69AC4C2D9C83L, + 0x129B69070816E2FDL, 0xF5185489D68AE39CL, 0x1DC574D80CF16B2FL, 0xEE8D540FBDAB05C6L, + 0x17D12A4670C1228CL, 0xBED77672FE226B05L, 0x130DBB6B8D674ED6L, 0xFF12C528CB4EBC04L, + 0x1E7C5F127BD87E24L, 0xCB513B74787DF9A0L, 0x18637F41FCAD31B7L, 0x090DC929F9FE614DL, + 0x1382CC34CA2427C5L, 0xA0D7D42194CB810AL, 0x1F37AD21436D0C6FL, 0x67BFB9CF5478CE77L, + 0x18F9574DCF8A7059L, 0x1FCC94A5DD2D71F9L, 0x13FAAC3E3FA1F37AL, 0x7FD6DD517DBDF4C7L, + 0x1FF779FD329CB8C3L, 0xFFBE2EE8C92FEE0BL, 0x1992C7FDC216FA36L, 0x6631BF20A0F324D6L, + 0x14756CCB01ABFB5EL, 0xB827CC1A1A5C1D78L, 0x105DF0A267BCC918L, 0x935309AE7B7CE460L, + 0x1A2FE76A3F9474F4L, 0x1EEB42B0C594A099L, 0x14F31F8832DD2A5CL, 0xE58902270476E6E1L, + 0x10C27FA028B0EEB0L, 0xB7A0CE859D2BEBE7L, 0x1AD0CC33744E4AB4L, 0x59014A6F61DFDFD8L, + 0x1573D68F903EA229L, 0xE0CDD525E7E64CADL, 0x11297872D9CBB4EEL, 0x4D7177518651D6F1L, + 0x1B758D848FAC54B0L, 0x7BE8BEE8D6E957E8L, 0x15F7A46A0C89DD59L, 0xFCBA3253DF211320L, + 0x1192E9EE706E4AAEL, 0x63C8284318E74280L, 0x1C1E43171A4A1117L, 0x060D0D3827D86A66L, + 0x167E9C127B6E7412L, 0x6B3DA42CECAD21EBL, 0x11FEE341FC585CDBL, 0x88FE1CF0BD574E56L, + 0x1CCB0536608D615FL, 0x419694B462254A23L, 0x1708D0F84D3DE77FL, 0x67ABAA29E81DD4E9L, + 0x126D73F9D764B932L, 0xB95621BB2017DD87L, 0x1D7BECC2F23AC1EAL, 0xC223692B668C95A5L, + 0x179657025B6234BBL, 0xCE82BA891ED6DE1DL, 0x12DEAC01E2B4F6FCL, 0xA53562074BDF1818L, + 0x1E3113363787F194L, 0x3B889CD87964F359L, 0x18274291C6065ADCL, 0xFC6D4A46C783F5E1L, + 0x13529BA7D19EAF17L, 0x30576E9F06032B1AL, 0x1EEA92A61C311825L, 0x1A257DCB3CD1DE90L, + 0x18BBA884E35A79B7L, 0x481DFE3C30A7E540L, 0x13C9539D82AEC7C5L, 0xD34B31C9C0865100L, + 0x1FA885C8D117A609L, 0x5211E942CDA3B4CDL, 0x19539E3A40DFB807L, 0x74DB21023E1C90A4L, + 0x1442E4FB67196005L, 0xF715B401CB4A0D50L, 0x103583FC527AB337L, 0xF8DE299B09080AA7L, + 0x19EF3993B72AB859L, 0x8E304291A80CDDD7L, 0x14BF6142F8EEF9E1L, 0x3E8D020E200A4B13L, + 0x10991A9BFA58C7E7L, 0x653D9B3E80083C0FL, 0x1A8E90F9908E0CA5L, 0x6EC8F864000D2CE4L, + 0x153EDA614071A3B7L, 0x8BD3F9E999A423EAL, 0x10FF151A99F482F9L, 0x3CA994BAE1501CBBL, + 0x1B31BB5DC320D18EL, 0xC775BAC49BB3612BL, 0x15C162B168E70E0BL, 0xD2C4956A16291A89L, + 0x11678227871F3E6FL, 0xDBD0778811BA7BA1L, 0x1BD8D03F3E9863E6L, 0x2C80BF401C5D929BL, + 0x16470CFF6546B651L, 0xBD33CC3349E47549L, 0x11D270CC51055EA7L, 0xCA8FD68F6E505DD4L, + 0x1C83E7AD4E6EFDD9L, 0x4419574BE3B3C953L, 0x16CFEC8AA52597E1L, 0x0347790982F63AA9L, + 0x123FF06EEA847980L, 0xCF6C60D468C4FBBAL, 0x1D331A4B10D3F59AL, 0xE57A34870E07F92AL, + 0x175C1508DA432AE2L, 0x512E906C0B399422L, 0x12B010D3E1CF5581L, 0xDA8BA6BCD5C7A9B5L, + 0x1DE6815302E5559CL, 0x90DF712E22D90F87L, 0x17EB9AA8CF1DDE16L, 0xDA4C5A8B4F140C6CL, + 0x1322E220A5B17E78L, 0xAEA37BA2A5A9A38AL, 0x1E9E369AA2B59727L, 0x7DD25F6AA2A905A9L, + 0x187E92154EF7AC1FL, 0x97DB7F888220D154L, 0x139874DDD8C6234CL, 0x797C6606CE80A777L, + 0x1F5A549627A36BADL, 0x8F2D700AE4010BF1L, 0x191510781FB5EFBEL, 0x0C2459A25000D65AL, + 0x1410D9F9B2F7F2FEL, 0x701D1481D99A4515L, 0x100D7B2E28C65BFEL, 0xC017439B147B6A77L, + 0x19AF2B7D0E0A2CCAL, 0xCCF205C4ED9243F2L, 0x148C22CA71A1BD6FL, 0x0A5B37D0BE0E9CC2L, + 0x10701BD527B4978CL, 0x0848F973CB3EE3CEL, 0x1A4CF9550C5425ACL, 0xDA0E5BEC78649FB0L, + 0x150A6110D6A9B7BDL, 0x7B3EAFF060507FC0L, 0x10D51A73DEEE2C97L, 0x95CBBFF380406633L, + 0x1AEE90B964B04758L, 0xEFAC665266CD7052L, 0x158BA6FAB6F36C47L, 0x2623850EB8A459DBL, + 0x113C85955F29236CL, 0x1E82D0D893B6AE49L, 0x1B9408EEFEA838ACL, 0xFD9E1AF41F8AB075L, + 0x16100725988693BDL, 0x97B1AF29B2D559F7L, 0x11A66C1E139EDC97L, 0xAC8E25BAF5777B2CL, + 0x1C3D79C9B8FE2DBFL, 0x7A7D092B2258C513L, 0x169794A160CB57CCL, 0x61FDA0EF4EAD6A76L, + 0x1212DD4DE7091309L, 0xE7FE1A590BBDEEC5L, 0x1CEAFBAFD80E84DCL, 0xA6635D5B45FCB13AL, + 0x172262F3133ED0B0L, 0x851C4AAF6B308DC8L, 0x1281E8C275CBDA26L, 0xD0E36EF2BC26D7D4L, + 0x1D9CA79D894629D7L, 0xB49F17EAC6A48C86L, 0x17B08617A104EE46L, 0x2A18DFEF0550706BL, + 0x12F39E794D9D8B6BL, 0x54E0B3259DD9F389L, 0x1E5297287C2F4578L, 0x87CDEB6F62F65274L, + 0x18421286C9BF6AC6L, 0xD30B22BF825EA85DL, 0x13680ED23AFF889FL, 0x0F3C1BCC684BB9E4L, + 0x1F0CE4839198DA98L, 0x18602C7A4079296DL, 0x18D71D360E13E213L, 0x46B356C833942124L, + 0x13DF4A91A4DCB4DCL, 0x388F78A029434DB6L, 0x1FCBAA82A1612160L, 0x5A7F2766A86BAF8AL, + 0x196FBB9BB44DB44DL, 0x153285EBB9EFBFA2L, 0x145962E2F6A4903DL, 0xAA8ED189618C994EL, + 0x1047824F2BB6D9CAL, 0xEED8A7A11AD6E10CL, 0x1A0C03B1DF8AF611L, 0x7E27729B5E249B45L, + 0x14D6695B193BF80DL, 0xFE85F549181D4904L, 0x10AB877C142FF9A4L, 0xCB9E5DD4134AA0D0L, + 0x1AAC0BF9B9E65C3AL, 0xDF63C9535211014DL, 0x15566FFAFB1EB02FL, 0x191CA10F74DA6771L, + 0x1111F32F2F4BC025L, 0xADB080D92A4852C1L, 0x1B4FEB7EB212CD09L, 0x15E7348EAA0D5134L, + 0x15D98932280F0A6DL, 0xAB1F5D3EEE710DC4L, 0x117AD428200C0857L, 0xBC1917658B8DA49DL, + 0x1BF7B9D9CCE00D59L, 0x2CF4F23C127C3A94L, 0x165FC7E170B33DE0L, 0xF0C3F4FCDB969543L, + 0x11E6398126F5CB1AL, 0x5A365D9716121103L, 0x1CA38F350B22DE90L, 0x9056FC24F01CE804L, + 0x16E93F5DA2824BA6L, 0xD9DF301D8CE3ECD0L, 0x125432B14ECEA2EBL, 0xE17F59B13D8323DAL, + 0x1D53844EE47DD179L, 0x68CBC2B52F38395CL, 0x177603725064A794L, 0x53D6355DBF602DE3L, + 0x12C4CF8EA6B6EC76L, 0xA9782AB165E68B1CL, 0x1E07B27DD78B13F1L, 0x0F26AAB56FD744FAL, + 0x18062864AC6F4327L, 0x3F52222ABFDF6A62L, 0x1338205089F29C1FL, 0x65DB4E88997F884EL, + 0x1EC033B40FEA9365L, 0x6FC54A7428CC0D4AL, 0x1899C2F673220F84L, 0x596AA1F68709A43BL, + 0x13AE3591F5B4D936L, 0xADEEE7F86C07B696L, 0x1F7D228322BAF524L, 0x497E3FF3E00C5756L, + 0x1930E868E89590E9L, 0xD464FFF64CD6AC45L, 0x14272053ED4473EEL, 0x4383FFF83D7889D1L, + 0x101F4D0FF1038FF1L, 0xCF9CCCC69793A174L, 0x19CBAE7FE805B31CL, 0x7F6147A425B90252L, + 0x14A2F1FFECD15C16L, 0xCC4DD2E9B7C7350FL, 0x10825B3323DAB012L, 0x3D0B0F215FD290D9L, + 0x1A6A2B85062AB350L, 0x61AB4B689950E7C1L, 0x1521BC6A6B555C40L, 0x4E22A2BA1440B967L, + 0x10E7C9EEBC4449CDL, 0x0B4EE894DD009453L, 0x1B0C764AC6D3A948L, 0x1217DA87C800ED51L, + 0x15A391D56BDC876CL, 0xDB46486CA000BDDAL, 0x114FA7DDEFE39F8AL, 0x490506BD4CCD64AFL, + 0x1BB2A62FE638FF43L, 0xA8080AC87AE23AB1L, 0x162884F31E93FF69L, 0x5339A239FBE82EF4L, + 0x11BA03F5B20FFF87L, 0x75C7B4FB2FECF25DL, 0x1C5CD322B67FFF3FL, 0x22D92191E647EA2EL, + 0x16B0A8E891FFFF65L, 0xB57A8141850654F2L, 0x1226ED86DB3332B7L, 0xC4620101373843F5L, + 0x1D0B15A491EB8459L, 0x3A366801F1F39FEEL, 0x173C115074BC69E0L, 0xFB5EB99B27F6198BL, + 0x129674405D6387E7L, 0x2F7EFAE2865E7AD6L, 0x1DBD86CD6238D971L, 0xE597F7D0D6FD9156L, + 0x17CAD23DE82D7AC1L, 0x8479930D78CADAABL, 0x1308A831868AC89AL, 0xD06142712D6F1556L, + 0x1E74404F3DAADA91L, 0x4D686A4EAF182222L, 0x185D003F6488AEDAL, 0xA453883EF279B4E8L, + 0x137D99CC506D58AEL, 0xE9DC6CFF28615D87L, 0x1F2F5C7A1A488DE4L, 0xA960AE650D6895A4L, + 0x18F2B061AEA07183L, 0xBAB3BEB73DED4483L, 0x13F559E7BEE6C136L, 0x2EF6322C318A9D36L + }; + } + + private static long[] initPow5Split() { + return new long[]{ + 0x1000000000000000L, 0x0000000000000000L, 0x1400000000000000L, 0x0000000000000000L, + 0x1900000000000000L, 0x0000000000000000L, 0x1F40000000000000L, 0x0000000000000000L, + 0x1388000000000000L, 0x0000000000000000L, 0x186A000000000000L, 0x0000000000000000L, + 0x1E84800000000000L, 0x0000000000000000L, 0x1312D00000000000L, 0x0000000000000000L, + 0x17D7840000000000L, 0x0000000000000000L, 0x1DCD650000000000L, 0x0000000000000000L, + 0x12A05F2000000000L, 0x0000000000000000L, 0x174876E800000000L, 0x0000000000000000L, + 0x1D1A94A200000000L, 0x0000000000000000L, 0x12309CE540000000L, 0x0000000000000000L, + 0x16BCC41E90000000L, 0x0000000000000000L, 0x1C6BF52634000000L, 0x0000000000000000L, + 0x11C37937E0800000L, 0x0000000000000000L, 0x16345785D8A00000L, 0x0000000000000000L, + 0x1BC16D674EC80000L, 0x0000000000000000L, 0x1158E460913D0000L, 0x0000000000000000L, + 0x15AF1D78B58C4000L, 0x0000000000000000L, 0x1B1AE4D6E2EF5000L, 0x0000000000000000L, + 0x10F0CF064DD59200L, 0x0000000000000000L, 0x152D02C7E14AF680L, 0x0000000000000000L, + 0x1A784379D99DB420L, 0x0000000000000000L, 0x108B2A2C28029094L, 0x0000000000000000L, + 0x14ADF4B7320334B9L, 0x0000000000000000L, 0x19D971E4FE8401E7L, 0x4000000000000000L, + 0x1027E72F1F128130L, 0x8800000000000000L, 0x1431E0FAE6D7217CL, 0xAA00000000000000L, + 0x193E5939A08CE9DBL, 0xD480000000000000L, 0x1F8DEF8808B02452L, 0xC9A0000000000000L, + 0x13B8B5B5056E16B3L, 0xBE04000000000000L, 0x18A6E32246C99C60L, 0xAD85000000000000L, + 0x1ED09BEAD87C0378L, 0xD8E6400000000000L, 0x13426172C74D822BL, 0x878FE80000000000L, + 0x1812F9CF7920E2B6L, 0x6973E20000000000L, 0x1E17B84357691B64L, 0x03D0DA8000000000L, + 0x12CED32A16A1B11EL, 0x8262889000000000L, 0x178287F49C4A1D66L, 0x22FB2AB400000000L, + 0x1D6329F1C35CA4BFL, 0xABB9F56100000000L, 0x125DFA371A19E6F7L, 0xCB54395CA0000000L, + 0x16F578C4E0A060B5L, 0xBE2947B3C8000000L, 0x1CB2D6F618C878E3L, 0x2DB399A0BA000000L, + 0x11EFC659CF7D4B8DL, 0xFC90400474400000L, 0x166BB7F0435C9E71L, 0x7BB4500591500000L, + 0x1C06A5EC5433C60DL, 0xDAA16406F5A40000L, 0x118427B3B4A05BC8L, 0xA8A4DE8459868000L, + 0x15E531A0A1C872BAL, 0xD2CE16256FE82000L, 0x1B5E7E08CA3A8F69L, 0x87819BAECBE22800L, + 0x111B0EC57E6499A1L, 0xF4B1014D3F6D5900L, 0x1561D276DDFDC00AL, 0x71DD41A08F48AF40L, + 0x1ABA4714957D300DL, 0x0E549208B31ADB10L, 0x10B46C6CDD6E3E08L, 0x28F4DB456FF0C8EAL, + 0x14E1878814C9CD8AL, 0x33321216CBECFB25L, 0x1A19E96A19FC40ECL, 0xBFFE969C7EE839EEL, + 0x105031E2503DA893L, 0xF7FF1E21CF512435L, 0x14643E5AE44D12B8L, 0xF5FEE5AA43256D42L, + 0x197D4DF19D605767L, 0x337E9F14D3EEC893L, 0x1FDCA16E04B86D41L, 0x005E46DA08EA7AB7L, + 0x13E9E4E4C2F34448L, 0xA03AEC4845928CB3L, 0x18E45E1DF3B0155AL, 0xC849A75A56F72FDFL, + 0x1F1D75A5709C1AB1L, 0x7A5C1130ECB4FBD7L, 0x13726987666190AEL, 0xEC798ABE93F11D66L, + 0x184F03E93FF9F4DAL, 0xA797ED6E38ED64C0L, 0x1E62C4E38FF87211L, 0x517DE8C9C728BDF0L, + 0x12FDBB0E39FB474AL, 0xD2EEB17E1C7976B6L, 0x17BD29D1C87A191DL, 0x87AA5DDDA397D463L, + 0x1DAC74463A989F64L, 0xE994F5550C7DC97CL, 0x128BC8ABE49F639FL, 0x11FD195527CE9DEEL, + 0x172EBAD6DDC73C86L, 0xD67C5FAA71C24569L, 0x1CFA698C95390BA8L, 0x8C1B77950E32D6C3L, + 0x121C81F7DD43A749L, 0x57912ABD28DFC63AL, 0x16A3A275D494911BL, 0xAD75756C7317B7C9L, + 0x1C4C8B1349B9B562L, 0x98D2D2C78FDDA5BBL, 0x11AFD6EC0E14115DL, 0x9F83C3BCB9EA8795L, + 0x161BCCA7119915B5L, 0x0764B4ABE865297AL, 0x1BA2BFD0D5FF5B22L, 0x493DE1D6E27E73D8L, + 0x1145B7E285BF98F5L, 0x6DC6AD264D8F0867L, 0x159725DB272F7F32L, 0xC938586FE0F2CA81L, + 0x1AFCEF51F0FB5EFFL, 0x7B866E8BD92F7D21L, 0x10DE1593369D1B5FL, 0xAD34051767BDAE35L, + 0x15159AF804446237L, 0x9881065D41AD19C2L, 0x1A5B01B605557AC5L, 0x7EA147F492186033L, + 0x1078E111C3556CBBL, 0x6F24CCF8DB4F3C20L, 0x14971956342AC7EAL, 0x4AEE003712230B28L, + 0x19BCDFABC13579E4L, 0xDDA98044D6ABCDF1L, 0x10160BCB58C16C2FL, 0x0A89F02B062B60B7L, + 0x141B8EBE2EF1C73AL, 0xCD2C6C35C7B638E5L, 0x1922726DBAAE3909L, 0x8077874339A3C71EL, + 0x1F6B0F092959C74BL, 0xE0956914080CB8E5L, 0x13A2E965B9D81C8FL, 0x6C5D61AC8507F38FL, + 0x188BA3BF284E23B3L, 0x4774BA17A649F073L, 0x1EAE8CAEF261ACA0L, 0x1951E89D8FDC6C90L, + 0x132D17ED577D0BE4L, 0x0FD3316279E9C3DAL, 0x17F85DE8AD5C4EDDL, 0x13C7FDBB186434D0L, + 0x1DF67562D8B36294L, 0x58B9FD29DE7D4204L, 0x12BA095DC7701D9CL, 0xB7743E3A2B0E4943L, + 0x17688BB5394C2503L, 0xE5514DC8B5D1DB93L, 0x1D42AEA2879F2E44L, 0xDEA5A13AE3465278L, + 0x1249AD2594C37CEBL, 0x0B2784C4CE0BF38BL, 0x16DC186EF9F45C25L, 0xCDF165F6018EF06EL, + 0x1C931E8AB871732FL, 0x416DBF7381F2AC89L, 0x11DBF316B346E7FDL, 0x88E497A83137ABD6L, + 0x1652EFDC6018A1FCL, 0xEB1DBD923D8596CBL, 0x1BE7ABD3781ECA7CL, 0x25E52CF6CCE6FC7EL, + 0x1170CB642B133E8DL, 0x97AF3C1A40105DCFL, 0x15CCFE3D35D80E30L, 0xFD9B0B20D0147543L, + 0x1B403DCC834E11BDL, 0x3D01CDE904199293L, 0x1108269FD210CB16L, 0x462120B1A28FFB9CL, + 0x154A3047C694FDDBL, 0xD7A968DE0B33FA83L, 0x1A9CBC59B83A3D52L, 0xCD93C3158E00F924L, + 0x10A1F5B813246653L, 0xC07C59ED78C09BB7L, 0x14CA732617ED7FE8L, 0xB09B7068D6F0C2A4L, + 0x19FD0FEF9DE8DFE2L, 0xDCC24C830CACF34DL, 0x103E29F5C2B18BEDL, 0xC9F96FD1E7EC1810L, + 0x144DB473335DEEE9L, 0x3C77CBC661E71E14L, 0x1961219000356AA3L, 0x8B95BEB7FA60E599L, + 0x1FB969F40042C54CL, 0x6E7B2E65F8F91EFFL, 0x13D3E2388029BB4FL, 0xC50CFCFFBB9BB360L, + 0x18C8DAC6A0342A23L, 0xB6503C3FAA82A038L, 0x1EFB1178484134ACL, 0xA3E44B4F95234845L, + 0x135CEAEB2D28C0EBL, 0xE66EAF11BD360D2CL, 0x183425A5F872F126L, 0xE00A5AD62C839076L, + 0x1E412F0F768FAD70L, 0x980CF18BB7A47494L, 0x12E8BD69AA19CC66L, 0x5F0816F752C6C8DDL, + 0x17A2ECC414A03F7FL, 0xF6CA1CB527787B14L, 0x1D8BA7F519C84F5FL, 0xF47CA3E2715699D8L, + 0x127748F9301D319BL, 0xF8CDE66D86D62027L, 0x17151B377C247E02L, 0xF7016008E88BA831L, + 0x1CDA62055B2D9D83L, 0xB4C1B80B22AE923DL, 0x12087D4358FC8272L, 0x50F91306F5AD1B66L, + 0x168A9C942F3BA30EL, 0xE53757C8B3186240L, 0x1C2D43B93B0A8BD2L, 0x9E852DBADFDE7AD0L, + 0x119C4A53C4E69763L, 0xA3133C94CBEB0CC2L, 0x16035CE8B6203D3CL, 0x8BD80BB9FEE5CFF2L, + 0x1B843422E3A84C8BL, 0xAECE0EA87E9F43EFL, 0x1132A095CE492FD7L, 0x4D40C9294F238A76L, + 0x157F48BB41DB7BCDL, 0x2090FB73A2EC6D13L, 0x1ADF1AEA12525AC0L, 0x68B53A508BA78857L, + 0x10CB70D24B7378B8L, 0x417144725748B537L, 0x14FE4D06DE5056E6L, 0x51CD958EED1AE284L, + 0x1A3DE04895E46C9FL, 0xE640FAF2A8619B25L, 0x1066AC2D5DAEC3E3L, 0xEFE89CD7A93D00F8L, + 0x14805738B51A74DCL, 0xEBE2C40D938C4135L, 0x19A06D06E2611214L, 0x26DB7510F86F5182L, + 0x100444244D7CAB4CL, 0x9849292A9B4592F2L, 0x1405552D60DBD61FL, 0xBE5B73754216F7AEL, + 0x1906AA78B912CBA7L, 0xADF25052929CB599L, 0x1F485516E7577E91L, 0x996EE4673743E300L, + 0x138D352E5096AF1AL, 0xFFE54EC0828A6DE0L, 0x18708279E4BC5AE1L, 0xBFDEA270A32D0958L, + 0x1E8CA3185DEB719AL, 0x2FD64B0CCBF84BAEL, 0x1317E5EF3AB32700L, 0x5DE5EEE7FF7B2F4DL, + 0x17DDDF6B095FF0C0L, 0x755F6AA1FF59FB20L, 0x1DD55745CBB7ECF0L, 0x92B7454A7F3079E8L, + 0x12A5568B9F52F416L, 0x5BB28B4E8F7E4C31L, 0x174EAC2E8727B11BL, 0xF29F2E22335DDF3DL, + 0x1D22573A28F19D62L, 0xEF46F9AAC035570CL, 0x123576845997025DL, 0xD58C5C0AB8215668L, + 0x16C2D4256FFCC2F5L, 0x4AEF730D6629AC02L, 0x1C73892ECBFBF3B2L, 0x9DAB4FD0BFB41702L, + 0x11C835BD3F7D784FL, 0xA28B11E277D08E61L, 0x163A432C8F5CD663L, 0x8B2DD65B15C4B1FAL, + 0x1BC8D3F7B3340BFCL, 0x6DF94BF1DB35DE78L, 0x115D847AD000877DL, 0xC4BBCF772901AB0BL, + 0x15B4E5998400A95DL, 0x35EAC354F34215CEL, 0x1B221EFFE500D3B4L, 0x8365742A30129B41L, + 0x10F5535FEF208450L, 0xD21F689A5E0BA109L, 0x1532A837EAE8A565L, 0x06A742C0F58E894BL, + 0x1A7F5245E5A2CEBEL, 0x4851137132F22B9EL, 0x108F936BAF85C136L, 0xED32AC26BFD75B43L, + 0x14B378469B673184L, 0xA87F57306FCD3213L, 0x19E056584240FDE5L, 0xD29F2CFC8BC07E98L, + 0x102C35F729689EAFL, 0xA3A37C1DD7584F1FL, 0x14374374F3C2C65BL, 0x8C8C5B254D2E62E7L, + 0x1945145230B377F2L, 0x6FAF71EEA079FBA0L, 0x1F965966BCE055EFL, 0x0B9B4E6A48987A88L, + 0x13BDF7E0360C35B5L, 0x674111026D5F4C95L, 0x18AD75D8438F4322L, 0xC111554308B71FBBL, + 0x1ED8D34E547313EBL, 0x7155AA93CAE4E7A9L, 0x13478410F4C7EC73L, 0x26D58A9C5ECF10CAL, + 0x1819651531F9E78FL, 0xF08AED437682D4FCL, 0x1E1FBE5A7E786173L, 0xECADA89454238A3BL, + 0x12D3D6F88F0B3CE8L, 0x73EC895CB4963665L, 0x1788CCB6B2CE0C22L, 0x90E7ABB3E1BBC3FEL, + 0x1D6AFFE45F818F2BL, 0x352196A0DA2AB4FEL, 0x1262DFEEBBB0F97BL, 0x0134FE24885AB11FL, + 0x16FB97EA6A9D37D9L, 0xC1823DADAA715D66L, 0x1CBA7DE5054485D0L, 0x31E2CD19150DB4C0L, + 0x11F48EAF234AD3A2L, 0x1F2DC02FAD2890F8L, 0x1671B25AEC1D888AL, 0xA6F9303B9872B536L, + 0x1C0E1EF1A724EAADL, 0x50B77C4A7E8F6283L, 0x1188D357087712ACL, 0x5272ADAE8F199D92L, + 0x15EB082CCA94D757L, 0x670F591A32E004F7L, 0x1B65CA37FD3A0D2DL, 0x40D32F60BF980634L, + 0x111F9E62FE44483CL, 0x4883FD9C77BF03E1L, 0x156785FBBDD55A4BL, 0x5AA4FD0395AEC4D9L, + 0x1AC1677AAD4AB0DEL, 0x314E3C447B1A760FL, 0x10B8E0ACAC4EAE8AL, 0xDED0E5AACCF089CAL, + 0x14E718D7D7625A2DL, 0x96851F15802CAC3CL, 0x1A20DF0DCD3AF0B8L, 0xFC2666DAE037D74BL, + 0x10548B68A044D673L, 0x9D980048CC22E68FL, 0x1469AE42C8560C10L, 0x84FE005AFF2BA033L, + 0x198419D37A6B8F14L, 0xA63D8071BEF6883FL, 0x1FE52048590672D9L, 0xCFCCE08E2EB42A4FL, + 0x13EF342D37A407C8L, 0x21E00C58DD309A71L, 0x18EB0138858D09BAL, 0x2A580F6F147CC10EL, + 0x1F25C186A6F04C28L, 0xB4EE134AD99BF151L, 0x137798F428562F99L, 0x7114CC0EC80176D3L, + 0x18557F31326BBB7FL, 0xCD59FF127A01D487L, 0x1E6ADEFD7F06AA5FL, 0xC0B07ED7188249A9L, + 0x1302CB5E6F642A7BL, 0xD86E4F466F516E0AL, 0x17C37E360B3D351AL, 0xCE89E3180B25C98CL, + 0x1DB45DC38E0C8261L, 0x822C5BDE0DEF3BEFL, 0x1290BA9A38C7D17CL, 0xF15BB96AC8B58576L, + 0x1734E940C6F9C5DCL, 0x2DB2A7C57AE2E6D3L, 0x1D022390F8B83753L, 0x391F51B6D99BA087L, + 0x1221563A9B732294L, 0x03B3931248014455L, 0x16A9ABC9424FEB39L, 0x04A077D6DA01956AL, + 0x1C5416BB92E3E607L, 0x45C895CC9081FAC4L, 0x11B48E353BCE6FC4L, 0x8B9D5D9FDA513CBBL, + 0x1621B1C28AC20BB5L, 0xAE84B507D0E58BE9L, 0x1BAA1E332D728EA3L, 0x1A25E249C51EEEE4L, + 0x114A52DFFC679925L, 0xF057AD6E1B33554EL, 0x159CE797FB817F6FL, 0x6C6D98C9A2002AA2L, + 0x1B04217DFA61DF4BL, 0x4788FEFC0A80354AL, 0x10E294EEBC7D2B8FL, 0x0CB59F5D8690214FL, + 0x151B3A2A6B9C7672L, 0xCFE30734E83429A2L, 0x1A6208B50683940FL, 0x83DBC9022241340BL, + 0x107D457124123C89L, 0xB2695DA15568C087L, 0x149C96CD6D16CBACL, 0x1F03B509AAC2F0A8L, + 0x19C3BC80C85C7E97L, 0x26C4A24C1573ACD2L, 0x101A55D07D39CF1EL, 0x783AE56F8D684C04L, + 0x1420EB449C8842E6L, 0x16499ECB70C25F04L, 0x19292615C3AA539FL, 0x9BDC067E4CF2F6C5L, + 0x1F736F9B3494E887L, 0x82D3081DE02FB477L, 0x13A825C100DD1154L, 0xB1C3E512AC1DD0CAL, + 0x18922F31411455A9L, 0xDE34DE57572544FDL, 0x1EB6BAFD91596B14L, 0x55C215ED2CEE963CL, + 0x133234DE7AD7E2ECL, 0xB5994DB43C151DE6L, 0x17FEC216198DDBA7L, 0xE2FFA1214B1A655FL, + 0x1DFE729B9FF15291L, 0xDBBF89699DE0FEB7L, 0x12BF07A143F6D39BL, 0x2957B5E202AC9F32L, + 0x176EC98994F48881L, 0xF3ADA35A8357C6FFL, 0x1D4A7BEBFA31AAA2L, 0x70990C31242DB8BEL, + 0x124E8D737C5F0AA5L, 0x865FA79EB69C9377L, 0x16E230D05B76CD4EL, 0xE7F791866443B855L, + 0x1C9ABD04725480A2L, 0xA1F575E7FD54A66AL, 0x11E0B622C774D065L, 0xA53969B0FE54E802L, + 0x1658E3AB7952047FL, 0x0E87C41D3DEA2203L, 0x1BEF1C9657A6859EL, 0xD229B5248D64AA83L, + 0x117571DDF6C81383L, 0x435A1136D85EEA92L, 0x15D2CE55747A1864L, 0x143095848E76A537L, + 0x1B4781EAD1989E7DL, 0x193CBAE5B2144E84L, 0x110CB132C2FF630EL, 0x2FC5F4CF8F4CB113L, + 0x154FDD7F73BF3BD1L, 0xBBB77203731FDD57L, 0x1AA3D4DF50AF0AC6L, 0x2AA54E844FE7D4ADL, + 0x10A6650B926D66BBL, 0xDAA75112B1F0E4ECL, 0x14CFFE4E7708C06AL, 0xD15125575E6D1E27L, + 0x1A03FDE214CAF085L, 0x85A56EAD360865B1L, 0x10427EAD4CFED653L, 0x7387652C41C53F8FL, + 0x14531E58A03E8BE8L, 0x50693E7752368F72L, 0x1967E5EEC84E2EE2L, 0x64838E1526C4334FL, + 0x1FC1DF6A7A61BA9AL, 0xFDA4719A70754023L, 0x13D92BA28C7D14A0L, 0xDE86C70086494816L, + 0x18CF768B2F9C59C9L, 0x162878C0A7DB9A1BL, 0x1F03542DFB83703BL, 0x5BB296F0D1D280A2L, + 0x1362149CBD322625L, 0x194F9E5683239065L, 0x183A99C3EC7EAFAEL, 0x5FA385EC23EC747FL, + 0x1E494034E79E5B99L, 0xF78C67672CE7919EL, 0x12EDC82110C2F940L, 0x3AB7C0A07C10BB03L, + 0x17A93A2954F3B790L, 0x4965B0C89B14E9C4L, 0x1D9388B3AA30A574L, 0x5BBF1CFAC1DA2434L, + 0x127C35704A5E6768L, 0xB957721CB92856A1L, 0x171B42CC5CF60142L, 0xE7AD4EA3E7726C49L, + 0x1CE2137F74338193L, 0xA198A24CE14F075BL, 0x120D4C2FA8A030FCL, 0x44FF65700CD16499L, + 0x16909F3B92C83D3BL, 0x563F3ECC1005BDBFL, 0x1C34C70A777A4C8AL, 0x2BCF0E7F14072D2FL, + 0x11A0FC668AAC6FD6L, 0x5B61690F6C847C3EL, 0x16093B802D578BCBL, 0xF239C35347A59B4DL, + 0x1B8B8A6038AD6EBEL, 0xEEC83428198F0220L, 0x1137367C236C6537L, 0x553D20990FF96154L, + 0x1585041B2C477E85L, 0x2A8C68BF53F7B9A9L, 0x1AE64521F7595E26L, 0x752F82EF28F5A813L, + 0x10CFEB353A97DAD8L, 0x093DB1D57999890CL, 0x1503E602893DD18EL, 0x0B8D1E4AD7FFEB4FL, + 0x1A44DF832B8D45F1L, 0x8E7065DD8DFFE623L, 0x106B0BB1FB384BB6L, 0xF9063FAA78BFEFD6L, + 0x1485CE9E7A065EA4L, 0xB747CF9516EFEBCBL, 0x19A742461887F64DL, 0xE519C37A5CABE6BEL, + 0x1008896BCF54F9F0L, 0xAF301A2C79EB7037L, 0x140AABC6C32A386CL, 0xDAFC20B798664C44L, + 0x190D56B873F4C688L, 0x11BB28E57E7FDF55L, 0x1F50AC6690F1F82AL, 0x1629F31EDE1FD72BL, + 0x13926BC01A973B1AL, 0x4DDA37F34AD3E67BL, 0x187706B0213D09E0L, 0xE150C5F01D88E01AL, + 0x1E94C85C298C4C59L, 0x19A4F76C24EB1820L, 0x131CFD3999F7AFB7L, 0xB0071AA39712EF14L, + 0x17E43C8800759BA5L, 0x9C08E14C7CD7AAD9L, 0x1DDD4BAA0093028FL, 0x030B199F9C0D958FL, + 0x12AA4F4A405BE199L, 0x61E6F003C1887D7AL, 0x1754E31CD072D9FFL, 0xBA60AC04B1EA9CD8L, + 0x1D2A1BE4048F907FL, 0xA8F8D705DE65440EL, 0x123A516E82D9BA4FL, 0xC99B8663AAFF4A89L, + 0x16C8E5CA239028E3L, 0xBC0267FC95BF1D2BL, 0x1C7B1F3CAC74331CL, 0xAB0301FBBB2EE475L, + 0x11CCF385EBC89FF1L, 0xEAE1E13D54FD4ECAL, 0x1640306766BAC7EEL, 0x659A598CAA3CA27CL, + 0x1BD03C81406979E9L, 0xFF00EFEFD4CBCB1BL, 0x116225D0C841EC32L, 0x3F6095F5E4FF5EF1L, + 0x15BAAF44FA52673EL, 0xCF38BB735E3F36ADL, 0x1B295B1638E7010EL, 0x8306EA5035CF0458L, + 0x10F9D8EDE39060A9L, 0x11E4527221A162B7L, 0x15384F295C7478D3L, 0x565D670EAA09BB65L, + 0x1A8662F3B3919708L, 0x2BF4C0D2548C2A3EL, 0x1093FDD8503AFE65L, 0x1B78F88374D79A67L, + 0x14B8FD4E6449BDFEL, 0x625736A4520D8101L, 0x19E73CA1FD5C2D7DL, 0xFAED044D6690E141L, + 0x103085E53E599C6EL, 0xBCD422B0601A8CC9L, 0x143CA75E8DF0038AL, 0x6C092B5C78212FFBL, + 0x194BD136316C046DL, 0x070B763396297BF9L, 0x1F9EC583BDC70588L, 0x48CE53C07BB3DAF7L, + 0x13C33B72569C6375L, 0x2D80F4584D5068DBL, 0x18B40A4EEC437C52L, 0x78E1316E60A48311L, + }; + } + + /** + * Computes (m * (mul0:mul1)) >> shift, where mul0 is the high 64 bits and + * mul1 is the low 64 bits of a 128-bit multiplier. + *

+ * The 192-bit product is: high0 * 2^128 + (low0 + high1) * 2^64 + low1. + * We extract the upper 128 bits and right-shift by (shift - 64). + */ + // Requires shift in [65, 127] (guaranteed by the Ryu algorithm). + private static long mulShift64(long m, long mul0, long mul1, int shift) { + // Math.multiplyHigh is signed; correct for unsigned by adding m + // when the other operand is negative (MSB set). + long high1 = Math.multiplyHigh(m, mul1) + ((mul1 >> 63) & m); + long low0 = m * mul0; + long high0 = Math.multiplyHigh(m, mul0) + ((mul0 >> 63) & m); + + long mid = low0 + high1; + long carry = Long.compareUnsigned(mid, low0) < 0 ? 1L : 0L; + long high = high0 + carry; + + int s = shift - 64; + return (high << (64 - s)) | (mid >>> s); + } + + // Requires p in [0, 63] because Java's << masks the shift count to 6 bits. + // Callers guarantee p < 63 via the surrounding `q < 63` guard. + private static boolean multipleOfPowerOf2(long value, int p) { + assert p >= 0 && p < 64; + return (value & ((1L << p) - 1)) == 0; + } + + private static boolean multipleOfPowerOf5(long value, int p) { + return pow5Factor(value) >= p; + } + + private static int pow5Factor(long value) { + int count = 0; + while (value > 0) { + if (value % 5 != 0) { + return count; + } + value /= 5; + count++; + } + return count; + } + + private static int pow5bits(int e) { + return ((e * 1217359) >>> 19) + 1; + } + + /** + * Converts a decoded IEEE 754 double to its shortest decimal representation. + * + * @param ieeeMantissa the 52-bit mantissa (without implicit leading 1) + * @param ieeeExponent the biased 11-bit exponent (0-2046; 0 means subnormal) + * @param e10Out single-element array; receives the decimal exponent e10 + * such that value = output * 10^e10 + * @return the decimal significand as a positive long (1-17 digits) + */ + static long d2d(long ieeeMantissa, int ieeeExponent, int[] e10Out) { + int e2; + long m2; + if (ieeeExponent == 0) { + // Subnormal: no implicit leading 1, exponent is 1 (not 0) + e2 = 1 - 1023 - 52; + m2 = ieeeMantissa; + } else { + e2 = ieeeExponent - 1023 - 52; + m2 = ieeeMantissa | (1L << 52); + } + + // Fast path for small integers: if the value is an exact integer in [1, 2^53), + // return it directly. This avoids the 4x-scaling issue in the main Ryu path. + if (e2 >= -52 && e2 <= 0) { + long mask = (1L << -e2) - 1; + if ((m2 & mask) == 0) { + long mantissa = m2 >>> -e2; + int exp = 0; + // Strip trailing decimal zeros + while (mantissa > 0) { + long q = mantissa / 10; + long r = mantissa - 10 * q; + if (r != 0) { + break; + } + mantissa = q; + exp++; + } + e10Out[0] = exp; + return mantissa; + } + } + + // Subtract 2 from e2 to account for the 4x scaling (mv = 4*m2 below). + // This ensures mv * 2^e2 == m2 * 2^(e2_original), so the decimal + // conversion produces the correct (non-4x) significand. + e2 -= 2; + + boolean acceptBounds = (m2 & 1) == 0; + + // Step 2: Determine the interval of valid decimal representations. + long mv = 4 * m2; + long mp = 4 * m2 + 2; + int mmShift = (ieeeMantissa != 0 || ieeeExponent <= 1) ? 1 : 0; + long mm = 4 * m2 - 1 - mmShift; + + // Step 3: Convert to a decimal power base using 128-bit arithmetic. + long vr, vp, vm; + int e10; + boolean vmIsTrailingZeros = false; + boolean vrIsTrailingZeros = false; + + if (e2 >= 0) { + int q = Math.max(0, ((e2 * 78913) >>> 18) - 1); + e10 = q; + int k = DOUBLE_POW5_INV_BITCOUNT + pow5bits(q) - 1; + int i = -e2 + q + k; + int idx = 2 * q; + long mul0 = DOUBLE_POW5_INV_SPLIT[idx]; + long mul1 = DOUBLE_POW5_INV_SPLIT[idx + 1]; + vr = mulShift64(mv, mul0, mul1, i); + vp = mulShift64(mp, mul0, mul1, i); + vm = mulShift64(mm, mul0, mul1, i); + + if (q <= 21) { + if (mv % 5 == 0) { + vrIsTrailingZeros = multipleOfPowerOf5(mv, q); + } else if (acceptBounds) { + vmIsTrailingZeros = multipleOfPowerOf5(mm, q); + } else { + if (multipleOfPowerOf5(mp, q)) { + vp--; + } + } + } + } else { + int q = Math.max(0, ((-e2 * 732923) >>> 20) - 1); + e10 = q + e2; + int i = -e2 - q; + int k = pow5bits(i) - DOUBLE_POW5_BITCOUNT; + int j = q - k; + int idx = 2 * i; + long mul0 = DOUBLE_POW5_SPLIT[idx]; + long mul1 = DOUBLE_POW5_SPLIT[idx + 1]; + vr = mulShift64(mv, mul0, mul1, j); + vp = mulShift64(mp, mul0, mul1, j); + vm = mulShift64(mm, mul0, mul1, j); + + if (q <= 1) { + vrIsTrailingZeros = true; + if (acceptBounds) { + vmIsTrailingZeros = mmShift == 1; + } else { + vp--; + } + } else if (q < 63) { + vrIsTrailingZeros = multipleOfPowerOf2(mv, q); + } + } + + // Step 4: Find the shortest decimal representation in the interval. + int removed = 0; + int lastRemovedDigit = 0; + + if (vmIsTrailingZeros || vrIsTrailingZeros) { + // General case with trailing zeros handling + for (; ; ) { + long vpDiv10 = vp / 10; + long vmDiv10 = vm / 10; + if (vpDiv10 <= vmDiv10) { + break; + } + long vrDiv10 = vr / 10; + boolean vmMod10IsZero = vm - 10 * vmDiv10 == 0; + vmIsTrailingZeros &= vmMod10IsZero; + vrIsTrailingZeros &= lastRemovedDigit == 0; + lastRemovedDigit = (int) (vr - 10 * vrDiv10); + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + + if (vmIsTrailingZeros) { + for (; ; ) { + long vmDiv10 = vm / 10; + if (vm - 10 * vmDiv10 != 0) { + break; + } + long vpDiv10 = vp / 10; + long vrDiv10 = vr / 10; + vrIsTrailingZeros &= lastRemovedDigit == 0; + lastRemovedDigit = (int) (vr - 10 * vrDiv10); + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + } + + if (vrIsTrailingZeros && lastRemovedDigit == 5 && (vr & 1) == 0) { + // Round even + lastRemovedDigit = 4; + } + + long output = vr; + if ((vr == vm && (!acceptBounds || !vmIsTrailingZeros)) || lastRemovedDigit >= 5) { + output++; + } + e10Out[0] = e10 + removed; + return output; + } else { + // Optimized loop when no trailing zeros + boolean roundUp = false; + long vpDiv100 = vp / 100; + long vmDiv100 = vm / 100; + if (vpDiv100 > vmDiv100) { + long vrDiv100 = vr / 100; + long vrMod100 = vr - 100 * vrDiv100; + roundUp = vrMod100 >= 50; + vr = vrDiv100; + vp = vpDiv100; + vm = vmDiv100; + removed += 2; + } + + for (; ; ) { + long vpDiv10 = vp / 10; + long vmDiv10 = vm / 10; + if (vpDiv10 <= vmDiv10) { + break; + } + long vrDiv10 = vr / 10; + roundUp = (vr - 10 * vrDiv10) >= 5; + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + + long output = vr + (vr == vm || roundUp ? 1 : 0); + e10Out[0] = e10 + removed; + return output; + } + } + + /** + * Returns the number of decimal digits in a value known to fit in 17 digits. + */ + static int decimalLength17(long v) { + if (v >= 10_000_000_000_000_000L) return 17; + if (v >= 1_000_000_000_000_000L) return 16; + if (v >= 100_000_000_000_000L) return 15; + if (v >= 10_000_000_000_000L) return 14; + if (v >= 1_000_000_000_000L) return 13; + if (v >= 100_000_000_000L) return 12; + if (v >= 10_000_000_000L) return 11; + if (v >= 1_000_000_000L) return 10; + if (v >= 100_000_000L) return 9; + if (v >= 10_000_000L) return 8; + if (v >= 1_000_000L) return 7; + if (v >= 100_000L) return 6; + if (v >= 10_000L) return 5; + if (v >= 1_000L) return 4; + if (v >= 100L) return 3; + if (v >= 10L) return 2; + return 1; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java new file mode 100644 index 000000000..05ff6c7ae --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java @@ -0,0 +1,187 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.Numbers; +import io.questdb.client.std.datetime.CommonUtils; +import io.questdb.client.std.str.Utf8StringSink; + +/** Schema-only UTC millisecond formatter matching the server timestamp-to-text conversion. */ +final class QwpSchemaTimestampFormatter { + private static final long DAY_MICROS = 86_400_000_000L; + private static final long AVG_YEAR_MICROS = (long) (365.2425 * DAY_MICROS); + private static final long HOUR_MICROS = 3_600_000_000L; + private static final long MILLI_MICROS = 1_000L; + private static final long MINUTE_MICROS = 60_000_000L; + private static final long SECOND_MICROS = 1_000_000L; + private static final long YEAR_MICROS_LEAP = 366 * DAY_MICROS; + private static final long YEAR_MICROS_NONLEAP = 365 * DAY_MICROS; + private static final long[] LEAP_MONTH_MICROS = new long[12]; + private static final long[] MONTH_MICROS = new long[12]; + + static { + long normal = 0; + long leap = 0; + for (int month = 0; month < 11; month++) { + normal += CommonUtils.DAYS_PER_MONTH[month] * DAY_MICROS; + leap += CommonUtils.getDaysPerMonth(month + 1, true) * DAY_MICROS; + MONTH_MICROS[month + 1] = normal; + LEAP_MONTH_MICROS[month + 1] = leap; + } + } + + private QwpSchemaTimestampFormatter() { + } + + static void appendDateTime(Utf8StringSink sink, long micros) { + if (micros == Long.MIN_VALUE) { + return; + } + int year = getYear(micros); + boolean leap = CommonUtils.isLeapYear(year); + int month = getMonthOfYear(micros, year, leap); + appendYear000(sink, year); + sink.putAscii('-'); + append0(sink, month); + sink.putAscii('-'); + append0(sink, getDayOfMonth(micros, year, month, leap)); + sink.putAscii('T'); + append0(sink, getHourOfDay(micros)); + sink.putAscii(':'); + append0(sink, getMinuteOfHour(micros)); + sink.putAscii(':'); + append0(sink, getSecondOfMinute(micros)); + sink.putAscii('.'); + append00(sink, getMillisOfSecond(micros)); + sink.putAscii('Z'); + } + + private static void append0(Utf8StringSink sink, int value) { + if (Math.abs(value) < 10) { + sink.putAscii('0'); + } + Numbers.append(sink, value); + } + + private static void append00(Utf8StringSink sink, int value) { + int absolute = Math.abs(value); + if (absolute < 10) { + sink.putAscii('0').putAscii('0'); + } else if (absolute < 100) { + sink.putAscii('0'); + } + Numbers.append(sink, value); + } + + private static void appendYear000(Utf8StringSink sink, int value) { + int absolute = Math.abs(value); + if (absolute < 10) { + sink.putAscii('0').putAscii('0').putAscii('0'); + } else if (absolute < 100) { + sink.putAscii('0').putAscii('0'); + } else if (absolute < 1000) { + sink.putAscii('0'); + } + Numbers.append(sink, value != 0 ? value : 1); + } + + private static int getDayOfMonth(long micros, int year, int month, boolean leap) { + long start = yearMicros(year, leap) + monthOfYearMicros(month, leap); + return (int) ((micros - start) / DAY_MICROS) + 1; + } + + private static int getHourOfDay(long micros) { + if (micros > -1) { + return (int) ((micros / HOUR_MICROS) % 24); + } + return 23 + (int) (((micros + 1) / HOUR_MICROS) % 24); + } + + private static int getMillisOfSecond(long micros) { + if (micros > -1) { + return (int) ((micros / MILLI_MICROS) % 1000); + } + return 999 + (int) (((micros + 1) / MILLI_MICROS) % 1000); + } + + private static int getMinuteOfHour(long micros) { + if (micros > -1) { + return (int) ((micros / MINUTE_MICROS) % 60); + } + return 59 + (int) (((micros + 1) / MINUTE_MICROS) % 60); + } + + // Keep the server's threshold-based month lookup: it is stable at the wrapped long extrema. + private static int getMonthOfYear(long micros, int year, boolean leap) { + int i = (int) (((micros - yearMicros(year, leap)) / 1000) >> 10); + if (leap) { + if (i < 182 * 84375) { + if (i < 91 * 84375) { + return i < 31 * 84375 ? 1 : i < 60 * 84375 ? 2 : 3; + } + return i < 121 * 84375 ? 4 : i < 152 * 84375 ? 5 : 6; + } + if (i < 274 * 84375) { + return i < 213 * 84375 ? 7 : i < 244 * 84375 ? 8 : 9; + } + return i < 305 * 84375 ? 10 : i < 335 * 84375 ? 11 : 12; + } + if (i < 181 * 84375) { + if (i < 90 * 84375) { + return i < 31 * 84375 ? 1 : i < 59 * 84375 ? 2 : 3; + } + return i < 120 * 84375 ? 4 : i < 151 * 84375 ? 5 : 6; + } + if (i < 273 * 84375) { + return i < 212 * 84375 ? 7 : i < 243 * 84375 ? 8 : 9; + } + return i < 304 * 84375 ? 10 : i < 334 * 84375 ? 11 : 12; + } + + private static int getSecondOfMinute(long micros) { + if (micros > -1) { + return (int) ((micros / SECOND_MICROS) % 60); + } + return 59 + (int) (((micros + 1) / SECOND_MICROS) % 60); + } + + // Source-compatible estimate and adjustment from the server Micros implementation. + private static int getYear(long micros) { + int yearsSinceEpoch = (int) (micros / AVG_YEAR_MICROS); + int estimate = 1970 + yearsSinceEpoch; + if (micros < 0 && estimate >= 1970) { + estimate = 1969; + } + boolean leap = CommonUtils.isLeapYear(estimate); + long start = yearMicros(estimate, leap); + long difference = micros - start; + if (difference < 0) { + estimate--; + } else if (difference >= (leap ? YEAR_MICROS_LEAP : YEAR_MICROS_NONLEAP)) { + estimate++; + } + return estimate; + } + + private static long monthOfYearMicros(int month, boolean leap) { + return (leap ? LEAP_MONTH_MICROS : MONTH_MICROS)[month - 1]; + } + + private static long yearMicros(int year, boolean leap) { + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + long days = year * 365L + (leapYears - 719527L); + long micros = days * DAY_MICROS; + return days < 0 && micros > 0 ? Long.MIN_VALUE : micros; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java new file mode 100644 index 000000000..a3dcbacf1 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java @@ -0,0 +1,599 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.NumericException; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.datetime.CommonUtils; + +import java.text.DateFormatSymbols; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneOffsetTransitionRule; +import java.time.zone.ZoneRules; +import java.time.zone.ZoneRulesProvider; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Source-faithful parser for the eleven formats accepted by QWP STRING-to-timestamp conversion. + * The grammar and computation mirror the fixed formats in the server's + * {@code MicrosFormatUtils} and {@code GenericMicrosFormat}; this is deliberately + * not a general date-format compiler. + */ +final class QwpSchemaTimestampParser { + private static final long DAY = 86_400_000_000L; + private static final int DAYS_0000_TO_1970 = 719527; + private static final long HOUR = 3_600_000_000L; + private static final long MINUTE = 60_000_000L; + private static final long SECOND = 1_000_000L; + private static final long WEEK = 7 * DAY; + private static final long[] MONTH_COMMON = monthStarts(false); + private static final long[] MONTH_LEAP = monthStarts(true); + private static final int REFERENCE_CENTURY = referenceCentury(); + + private QwpSchemaTimestampParser() { + } + + static long parse(CharSequence value) throws NumericException { + NumericException failure = NumericException.instance(); + try { + return parsePg(value); + } catch (NumericException ignored) { + } + try { + return parseIso(value); + } catch (NumericException ignored) { + } + throw failure; + } + + private static long parseIso(CharSequence value) throws NumericException { + int n = value.length(); + Parsed p = new Parsed(); + int pos = parseFixedYear(value, 0, n, p); + if (pos == n) { + return compute(p, null); + } + require(value, pos++, n, '-'); + if (pos + 3 <= n && value.charAt(pos) == 'W') { + p.week = Numbers.parseInt(value, pos + 1, pos + 3); + if (pos + 3 != n) { + throw NumericException.instance(); + } + return compute(p, null); + } + p.month = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return compute(p, null); + } + require(value, pos++, n, '-'); + p.day = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return compute(p, null); + } + require(value, pos++, n, 'T'); + p.hour = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return compute(p, null); + } + require(value, pos++, n, ':'); + p.minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + p.second = fixedInt(value, pos, pos += 2, n); + if (pos < n && value.charAt(pos) == '.') { + pos++; + try { + Parsed copy = new Parsed(p); + long parsed = greedyMillis(value, pos, n); + copy.millis = (int) parsed; + return compute(copy, ZoneMatcher.parse(value, pos + (int) (parsed >>> 32), n)); + } catch (NumericException ignored) { + } + if (pos + 6 <= n) { + try { + Parsed copy = new Parsed(p); + copy.millis = fixedInt(value, pos, pos + 3, n); + copy.micros = fixedInt(value, pos + 3, pos + 6, n); + return compute(copy, ZoneMatcher.parse(value, pos + 6, n)); + } catch (NumericException ignored) { + } + } + if (pos + 2 <= n) { + try { + fixedInt(value, pos, pos + 1, n); + Parsed copy = new Parsed(p); + long parsed = greedyMillis(value, pos + 1, n); + copy.millis = (int) parsed; + return compute(copy, ZoneMatcher.parse(value, pos + 1 + (int) (parsed >>> 32), n)); + } catch (NumericException ignored) { + } + } + // Final UTC_PATTERN fallback is fixed SSS. It intentionally follows + // the greedy S and SS forms, and therefore accepts parseInt forms + // such as +12 and 1_2 as well as a numeric-zone suffix after 3 chars. + if (pos + 3 <= n) { + try { + Parsed copy = new Parsed(p); + copy.millis = fixedInt(value, pos, pos + 3, n); + return compute(copy, ZoneMatcher.parse(value, pos + 3, n)); + } catch (NumericException ignored) { + } + } + throw NumericException.instance(); + } + return compute(p, ZoneMatcher.parse(value, pos, n)); + } + + private static long parsePg(CharSequence value) throws NumericException { + int n = value.length(); + int dash = indexOf(value, '-', value.length() > 0 && value.charAt(0) == '-' ? 1 : 0, n); + if (dash < 1) { + throw NumericException.instance(); + } + Parsed p = new Parsed(); + int year = greedyInt(value, 0, dash); + p.year = dash == 2 ? REFERENCE_CENTURY + year : year; + int pos = dash + 1; + p.month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + p.day = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ' '); + p.hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + p.minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + p.second = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '.'); + p.millis = fixedInt(value, pos, pos += 3, n); + return compute(p, ZoneMatcher.parse(value, pos, n)); + } + + private static long greedyMillis(CharSequence value, int lo, int hi) throws NumericException { + int p = lo; + boolean negative = p < hi && value.charAt(p) == '-'; + if (negative) { + p++; + } + int digitStart = p; + int parsed = 0; + while (p < hi && value.charAt(p) >= '0' && value.charAt(p) <= '9') { + parsed = parsed * 10 + value.charAt(p++) - '0'; + } + int width = p - lo; + if (p == digitStart || width > 3) { + throw NumericException.instance(); + } + while (width < 3) { + parsed *= 10; + width++; + } + return ((long) (p - lo) << 32) | ((negative ? -parsed : parsed) & 0xffffffffL); + } + + private static long compute(Parsed p, ZoneMatcher.Zone zone) throws NumericException { + boolean leap = CommonUtils.isLeapYear(p.year); + if (p.month < 1 || p.month > 12 || p.day < 1 || p.day > CommonUtils.getDaysPerMonth(p.month, leap) + || p.hour < 0 || p.hour > 24 || p.minute < 0 || p.minute > 59 + || p.second < 0 || p.second > 59 || (p.week != -1 && (p.week < 1 || p.week > weeks(p.year)))) { + throw NumericException.instance(); + } + if (p.week != -1) { + long first = yearMicros(p.year, CommonUtils.isLeapYear(p.year)) + + (p.week - 1L) * WEEK + isoYearDayOffset(p.year) * DAY; + int actualYear = year(first); + p.month = monthOfYear(first, actualYear, CommonUtils.isLeapYear(actualYear)); + p.year += p.week == 1 && isoYearDayOffset(p.year) < 0 ? -1 : 0; + p.day = dayOfMonth(first, p.year, p.month, CommonUtils.isLeapYear(p.year)); + // GenericMicrosFormat retains the originally parsed year's leap flag + // after an ISO week crosses a calendar-year boundary. + } + long out = yearMicros(p.year, leap) + monthMicros(p.month, leap) + (p.day - 1L) * DAY + + (p.hour % 24L) * HOUR + p.minute * MINUTE + p.second * SECOND + + p.millis * 1000L + p.micros; + return zone == null ? out : out - zone.offset(out, p.year); + } + + private static int parseFixedYear(CharSequence s, int pos, int hi, Parsed p) throws NumericException { + int width = pos < hi && s.charAt(pos) == '-' ? 5 : 4; + p.year = width == 5 + ? -fixedInt(s, pos + 1, pos + width, hi) + : fixedInt(s, pos, pos + width, hi); + return pos + width; + } + + private static int fixedInt(CharSequence s, int lo, int hi, int limit) throws NumericException { + if (hi > limit) { + throw NumericException.instance(); + } + return Numbers.parseInt(s, lo, hi); + } + + private static int greedyInt(CharSequence s, int lo, int hi) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); + } + int sign = 1; + if (s.charAt(lo) == '-') { + sign = -1; + lo++; + } + if (lo >= hi) { + throw NumericException.instance(); + } + long value = 0; + for (int i = lo; i < hi; i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + value = value * 10 + c - '0'; + if (value > (sign < 0 ? 2147483648L : Integer.MAX_VALUE)) { + throw NumericException.instance(); + } + } + return (int) (sign * value); + } + + private static int indexOf(CharSequence s, char c, int lo, int hi) { + for (int i = lo; i < hi; i++) { + if (s.charAt(i) == c) { + return i; + } + } + return -1; + } + + private static void require(CharSequence s, int pos, int hi, char c) throws NumericException { + if (pos >= hi || s.charAt(pos) != c) { + throw NumericException.instance(); + } + } + + private static int referenceCentury() { + int year = java.time.ZonedDateTime.now(ZoneOffset.UTC).getYear(); + int offset = year % 100; + return offset + 20 > 100 ? year - offset + 100 : year - offset; + } + + private static long[] monthStarts(boolean leap) { + long[] result = new long[12]; + for (int i = 1; i < 12; i++) { + result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap) * DAY; + } + return result; + } + + private static long monthMicros(int month, boolean leap) { + return (leap ? MONTH_LEAP : MONTH_COMMON)[month - 1]; + } + + private static long yearMicros(int year, boolean leap) { + // Preserve server Micros.yearMicros arithmetic, including unchecked + // multiplication and its asymmetric negative-overflow saturation. + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + long days = year * 365L + leapYears - DAYS_0000_TO_1970; + long micros = days * DAY; + return days < 0 && micros > 0 ? Long.MIN_VALUE : micros; + } + + private static int dayOfWeek(long micros) { + long d; + if (micros > -1) { + d = micros / DAY; + } else { + d = (micros - (DAY - 1)) / DAY; + if (d < -3) { + return 7 + (int) ((d + 4) % 7); + } + } + return 1 + (int) ((d + 3) % 7); + } + + private static int endYearDow(int year) { + return (year + Math.abs(year / 4) - Math.abs(year / 100) + Math.abs(year / 400)) % 7; + } + + private static int weeks(int year) { + return endYearDow(year) == 4 || endYearDow(year - 1) == 3 ? 53 : 52; + } + + private static int isoYearDayOffset(int year) { + int d = endYearDow(year - 1); + return (d <= 3 ? 0 : 7) - d; + } + + private static int monthOfYear(long micros, int year, boolean leap) { + long elapsed = micros - yearMicros(year, leap); + long[] starts = leap ? MONTH_LEAP : MONTH_COMMON; + int month = 12; + while (month > 1 && elapsed < starts[month - 1]) { + month--; + } + return month; + } + + private static int year(long micros) { + int estimate = 1970 + (int) (micros / 31_556_952_000_000L); + if (micros < 0 && estimate >= 1970) { + estimate = 1969; + } + boolean leap = CommonUtils.isLeapYear(estimate); + long diff = micros - yearMicros(estimate, leap); + if (diff < 0) { + return estimate - 1; + } + return diff >= (leap ? 366L : 365L) * DAY ? estimate + 1 : estimate; + } + + private static int dayOfMonth(long micros, int year, int month, boolean leap) { + return (int) ((micros - yearMicros(year, leap) - monthMicros(month, leap)) / DAY) + 1; + } + + private static long nextOrSame(long micros, int dow) { + int current = dayOfWeek(micros); + return current <= dow ? micros + (dow - current) * DAY : micros + (7 - current + dow) * DAY; + } + + private static long previousOrSame(long micros, int dow) { + int current = dayOfWeek(micros); + return current >= dow ? micros - (current - dow) * DAY : micros - (7 + current - dow) * DAY; + } + + private static final class Parsed { + private int day = 1; + private int hour; + private int micros; + private int millis; + private int minute; + private int month = 1; + private int second; + private int week = -1; + private int year; + + private Parsed() { + } + + private Parsed(Parsed that) { + day = that.day; + hour = that.hour; + micros = that.micros; + millis = that.millis; + minute = that.minute; + month = that.month; + second = that.second; + week = that.week; + year = that.year; + } + } + + private static final class ZoneMatcher { + private static final List TOKENS = tokens(); + + private static Zone parse(CharSequence value, int lo, int hi) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); + } + int minutes = numericOffset(value, lo, hi); + if (minutes != Integer.MIN_VALUE) { + return new Zone(minutes * MINUTE); + } + for (Token token : TOKENS) { + if (token.text.length() == hi - lo && regionMatches(value, lo, token.text)) { + return token.zone; + } + } + throw NumericException.instance(); + } + + private static int numericOffset(CharSequence s, int lo, int hi) { + if (hi - lo == 1 && (s.charAt(lo) == 'Z' || s.charAt(lo) == 'z')) { + return 0; + } + int p = lo; + boolean namedPrefix = startsWithIgnoreCase(s, p, hi, "UTC") || startsWithIgnoreCase(s, p, hi, "GMT"); + if (namedPrefix) { + p += 3; + } + boolean negative = false; + if (p < hi && (s.charAt(p) == '+' || s.charAt(p) == '-')) { + negative = s.charAt(p++) == '-'; + } else if (namedPrefix) { + return Integer.MIN_VALUE; + } + if (p + 2 > hi || !digit(s.charAt(p)) || !digit(s.charAt(p + 1))) { + return Integer.MIN_VALUE; + } + int hour = (s.charAt(p) - '0') * 10 + s.charAt(p + 1) - '0'; + p += 2; + int minute = 0; + if (p < hi) { + if (s.charAt(p) == ':') { + p++; + } + if (p + 2 > hi || !digit(s.charAt(p)) || !digit(s.charAt(p + 1))) { + return Integer.MIN_VALUE; + } + minute = (s.charAt(p) - '0') * 10 + s.charAt(p + 1) - '0'; + p += 2; + } + return p == hi && hour <= 23 && minute <= 59 + ? (negative ? -1 : 1) * (hour * 60 + minute) + : Integer.MIN_VALUE; + } + + private static List tokens() { + List result = new ArrayList<>(); + Set seen = new java.util.HashSet<>(); + Map rules = new HashMap<>(); + for (String id : ZoneRulesProvider.getAvailableZoneIds()) { + rules.put(id, new Zone(ZoneRulesProvider.getRules(id, true))); + } + for (Map.Entry alias : ZoneId.SHORT_IDS.entrySet()) { + if (!rules.containsKey(alias.getKey())) { + Zone target = rules.get(alias.getValue()); + if (target == null) { + target = new Zone(ZoneId.of(alias.getValue()).getRules()); + } + rules.put(alias.getKey(), target); + } + } + add(result, seen, rules.get("UTC"), "UTC"); + String[][] names = new DateFormatSymbols(Locale.ENGLISH).getZoneStrings(); + for (String[] row : names) { + if (row.length == 0 || !rules.containsKey(row[0])) { + continue; + } + for (String name : row) { + add(result, seen, rules.get(row[0]), name); + } + } + result.sort(Comparator.comparingInt((Token t) -> t.text.length()).reversed()); + return result; + } + + private static void add(List tokens, Set seen, Zone zone, String name) { + if (zone != null && name != null && !name.isEmpty() && seen.add(name)) { + tokens.add(new Token(name.toUpperCase(), zone)); + } + } + + private static boolean digit(char c) { + return c >= '0' && c <= '9'; + } + + private static boolean regionMatches(CharSequence value, int lo, String token) { + for (int i = 0; i < token.length(); i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != token.charAt(i)) { + return false; + } + } + return true; + } + + private static boolean startsWithIgnoreCase(CharSequence value, int lo, int hi, String prefix) { + if (hi - lo < prefix.length()) { + return false; + } + for (int i = 0; i < prefix.length(); i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != prefix.charAt(i)) { + return false; + } + } + return true; + } + + private static final class Token { + private final String text; + private final Zone zone; + + private Token(String text, Zone zone) { + this.text = text; + this.zone = zone; + } + } + + private static final class Zone { + private final long cutoff; + private final long fixedOffset; + private final long lastWall; + private final List recurring; + private final ZoneRules rules; + + private Zone(long fixedOffset) { + this.cutoff = Long.MIN_VALUE; + this.fixedOffset = fixedOffset; + this.lastWall = fixedOffset; + this.recurring = java.util.Collections.emptyList(); + this.rules = null; + } + + private Zone(ZoneRules rules) { + this.rules = rules; + List history = rules.getTransitions(); + cutoff = history.isEmpty() ? Long.MIN_VALUE : history.get(history.size() - 1).toEpochSecond() * SECOND; + recurring = rules.getTransitionRules(); + fixedOffset = rules.isFixedOffset() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() * SECOND : Long.MIN_VALUE; + lastWall = history.isEmpty() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() * SECOND + : history.get(history.size() - 1).getOffsetAfter().getTotalSeconds() * SECOND; + } + + private long offset(long epoch, int year) { + if (fixedOffset != Long.MIN_VALUE) { + return fixedOffset; + } + if (!recurring.isEmpty() && epoch > cutoff) { + // The server computes future rules with the parsed year, + // even when calendar arithmetic has wrapped the epoch. + long after = 0; + for (ZoneOffsetTransitionRule rule : recurring) { + long transition = transitionEpoch(rule, year); + long before = rule.getOffsetBefore().getTotalSeconds() * SECOND; + if (epoch < transition) { + return before; + } + after = rule.getOffsetAfter().getTotalSeconds() * SECOND; + } + return after; + } + if (epoch > cutoff) { + return lastWall; + } + Instant instant = Instant.ofEpochSecond(Math.floorDiv(epoch, SECOND), Math.floorMod(epoch, SECOND) * 1000); + return rules.getOffset(instant).getTotalSeconds() * SECOND; + } + + private static long transitionEpoch(ZoneOffsetTransitionRule rule, int year) { + boolean leap = CommonUtils.isLeapYear(year); + int month = rule.getMonth().getValue(); + int dom = rule.getDayOfMonthIndicator(); + int dow = rule.getDayOfWeek() == null ? -1 : rule.getDayOfWeek().getValue(); + long timestamp; + if (dom < 0) { + timestamp = yearMicros(year, leap) + monthMicros(month, leap) + + (CommonUtils.getDaysPerMonth(month, leap) + dom) * DAY + + rule.getLocalTime().getHour() * HOUR + rule.getLocalTime().getMinute() * MINUTE + + rule.getLocalTime().getSecond() * SECOND; + if (dow > -1) { + timestamp = previousOrSame(timestamp, dow); + } + } else { + timestamp = yearMicros(year, leap) + monthMicros(month, leap) + (dom - 1L) * DAY + + rule.getLocalTime().getHour() * HOUR + rule.getLocalTime().getMinute() * MINUTE + + rule.getLocalTime().getSecond() * SECOND; + if (dow > -1) { + timestamp = nextOrSame(timestamp, dow); + } + } + if (rule.isMidnightEndOfDay()) { + timestamp += DAY; + } + int before = rule.getOffsetBefore().getTotalSeconds(); + if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.UTC) { + timestamp += before * SECOND; + } else if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.STANDARD) { + timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * SECOND; + } + return timestamp - before * SECOND; + } + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 9f01f0ab8..f05689c7f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -33,6 +33,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -72,16 +73,17 @@ public class QwpTableBuffer implements QuietCloseable { private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access private int inProgressColumnCount; private int rowCount; + private QwpSchemaBinding schemaBinding; public QwpTableBuffer(String tableName) { this(tableName, null); } /** - * Use this constructor overload to allow writing to a symbol column. - * {@link ColumnBuffer#addSymbol(CharSequence)} needs the sender to - * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}, registering - * the symbol in the global dictionary shared with the server. + * Use this constructor overload to encode symbol columns with the sender's + * global dictionary IDs. {@link ColumnBuffer#addSymbol(CharSequence)} calls + * {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}; buffers built + * without a sender instead maintain a local per-column symbol dictionary. */ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.tableName = tableName; @@ -120,6 +122,7 @@ public void cancelCurrentRow() { * Frees all off-heap memory. */ public void clear() { + schemaBinding = null; for (int i = 0, n = columns.size(); i < n; i++) { columns.get(i).close(); } @@ -190,6 +193,11 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { return lookupColumn(name, type); } + ColumnBuffer getExistingInferredColumn(CharSequence name) { + int index = columnNameToIndex.get(name); + return index >= 0 ? columns.get(index) : null; + } + /** * Gets or creates a column with the given name and type. *

@@ -256,6 +264,23 @@ public String getTableName() { return tableName; } + public QwpSchemaBinding getSchemaBinding() { + return schemaBinding; + } + + void attachSchemaBinding(QwpSchemaBinding binding) { + if (schemaBinding != null || rowCount != 0 || hasInProgressRow() || columns.size() != 0) { + throw new IllegalStateException("schema binding requires an empty, unbound table buffer"); + } + schemaBinding = binding; + } + + void requireSchemaBinding(QwpSchemaBinding binding) { + if (schemaBinding != binding) { + throw new IllegalStateException("schema binding is no longer attached to the table buffer"); + } + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -638,6 +663,7 @@ public static class ColumnBuffer implements QuietCloseable { private OffHeapAppendMemory dataBuffer; // Decimal storage private byte decimalScale = -1; + private boolean schemaDecimalScaleLocked; private double[] doubleArrayData; // GeoHash precision (number of bits, 1-60) private int geohashPrecision = -1; @@ -699,10 +725,17 @@ public void addBinary(byte[] value) { throw new LineSenderException( "BINARY value cannot be null; mark the row null via the null bitmap instead"); } - if (value.length > 0) { - stringData.putBytes(value, 0, value.length); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (value.length > 0) { + stringData.putBytes(value, 0, value.length); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; size++; } @@ -720,10 +753,17 @@ public void addBinary(long ptr, long len) { if (len > 0 && ptr == 0) { throw new LineSenderException("BINARY pointer cannot be 0 for a non-empty value"); } - if (len > 0) { - stringData.putBlockOfBytes(ptr, len); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (len > 0) { + stringData.putBlockOfBytes(ptr, len); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; size++; } @@ -827,6 +867,77 @@ public void addDecimal64(Decimal64 value) { size++; } + public boolean addSchemaDecimal(Decimal value, int targetPrecision, int targetScale) throws NumericException { + value.toDecimal256(rescaleTemp); + return appendSchemaDecimal(targetPrecision, targetScale); + } + + public void addSchemaDecimalNull(int targetScale) { + lockSchemaDecimalScale(targetScale); + addNull(); + } + + public boolean addSchemaLongDecimal(long value, int targetPrecision, int targetScale) throws NumericException { + rescaleTemp.ofLong(value, 0); + return appendSchemaDecimal(targetPrecision, targetScale); + } + + public boolean addSchemaStringDecimal(CharSequence value, int targetPrecision, int targetScale) throws NumericException { + rescaleTemp.ofString(value, 0, value.length(), targetPrecision, targetScale, false, false); + if (rescaleTemp.isNull()) { + addSchemaDecimalNull(targetScale); + return true; + } + return appendSchemaDecimal(targetPrecision, targetScale); + } + + private boolean appendSchemaDecimal(int targetPrecision, int targetScale) throws NumericException { + if (rescaleTemp.getScale() != targetScale) { + rescaleTemp.rescale(targetScale); + } + if (!rescaleTemp.comparePrecision(targetPrecision)) { + return false; + } + int storageSizePow2; + switch (type) { + case TYPE_DECIMAL64: + storageSizePow2 = 3; + break; + case TYPE_DECIMAL128: + storageSizePow2 = 4; + break; + case TYPE_DECIMAL256: + storageSizePow2 = 5; + break; + default: + throw new IllegalStateException("not a decimal column: " + type); + } + if (!rescaleTemp.fitsInStorageSizePow2(storageSizePow2)) { + return false; + } + lockSchemaDecimalScale(targetScale); + switch (type) { + case TYPE_DECIMAL64: + dataBuffer.putLong(rescaleTemp.getLl()); + break; + case TYPE_DECIMAL128: + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + break; + case TYPE_DECIMAL256: + dataBuffer.putLong(rescaleTemp.getHh()); + dataBuffer.putLong(rescaleTemp.getHl()); + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + break; + default: + throw new AssertionError(); + } + valueCount++; + size++; + return true; + } + public void addDouble(double value) { dataBuffer.putDouble(value); valueCount++; @@ -946,6 +1057,13 @@ public void addFloat(float value) { * @param precision number of bits (1-60) */ public void addGeoHash(long value, int precision) { + initGeoHashPrecision(precision); + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void initGeoHashPrecision(int precision) { if (precision < 1 || precision > 60) { throw new LineSenderException("invalid GeoHash precision: " + precision + " (must be 1-60)"); } @@ -956,9 +1074,6 @@ public void addGeoHash(long value, int precision) { "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision ); } - dataBuffer.putLong(value); - valueCount++; - size++; } public void addInt(int value) { @@ -1180,10 +1295,17 @@ public void addString(CharSequence value) { ensureNullBitmapCapacity(size + 1); markNull(size); } else { - if (value != null) { - stringData.putUtf8(value); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (value != null) { + stringData.putUtf8(value); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; } size++; @@ -1491,7 +1613,9 @@ public void reset() { maxGlobalSymbolId = -1; arrayShapeOffset = 0; arrayDataOffset = 0; - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } // geohashPrecision is intentionally NOT cleared here. It is a // schema property, locked on first write and matched by the // server's auto-created GEOHASH(Nb) type, so preserving it across @@ -1610,7 +1734,9 @@ public void truncateTo(int newSize) { // through ColumnBuffer.reset() and intentionally preserves // geohashPrecision there to keep its server-locked value. if (newValueCount == 0) { - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } geohashPrecision = -1; maxGlobalSymbolId = -1; storeGlobalSymbolIdsOnly = false; @@ -1625,6 +1751,11 @@ public boolean usesNullBitmap() { return useNullBitmap; } + private void restoreStringAppendPositions(long dataOffset, long offsetsOffset) { + stringData.jumpTo(dataOffset); + stringOffsets.jumpTo(offsetsOffset); + } + private static int checkedElementCount(long product) { if (product > Integer.MAX_VALUE) { throw new LineSenderException("array too large: total element count exceeds int range"); @@ -1852,8 +1983,21 @@ private void markNull(int index) { hasNulls = true; } + private void lockSchemaDecimalScale(int targetScale) { + if (schemaDecimalScaleLocked) { + if ((decimalScale & 0xFF) != targetScale) { + throw new IllegalStateException("schema decimal scale changed"); + } + return; + } + decimalScale = (byte) targetScale; + schemaDecimalScaleLocked = true; + } + private void resetEmptyMetadata() { - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } geohashPrecision = -1; maxGlobalSymbolId = -1; storeGlobalSymbolIdsOnly = false; diff --git a/core/src/main/java/io/questdb/client/std/DecimalParser.java b/core/src/main/java/io/questdb/client/std/DecimalParser.java index 86fabdcbc..ece8c051d 100644 --- a/core/src/main/java/io/questdb/client/std/DecimalParser.java +++ b/core/src/main/java/io/questdb/client/std/DecimalParser.java @@ -35,7 +35,7 @@ *

  • QuestDB decimal suffix: "123.45m" or "123.45M"
    • *
  • Floating-point suffixes: "123.45f", "123.45d" (stripped during parsing)
    • *
  • Leading zeros: "00123.45" → 123.45
    • - *
  • Leading/trailing whitespace
    • + *
  • Leading ASCII spaces
    • * * * @see Decimal @@ -54,7 +54,7 @@ public final class DecimalParser { *

    * Parsing Process: *

      - *
    1. Strip suffixes (m, M, f, d) and leading/trailing whitespace
      2. + *
    2. Strip supported suffixes (m, M, f, d) and leading ASCII spaces
      3. *
    3. Parse sign (+/-) and detect special values (NaN/Infinity)
      4. *
    4. Remove leading zeros
      5. *
    5. Extract mantissa and locate decimal point
      6. @@ -163,12 +163,21 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // We do a first pass over the literal to ensure that the format is correct (numerical and at most 1 dot) and to // measure the given precision/scale. int dot = -1; + int leadingZeroes = 0; boolean digitFound = false; + boolean hasSignificantDigit = false; int digitLo = lo; for (; lo < hi; lo++) { char c = cs.charAt(lo); if (isDigit(c)) { digitFound = true; + if (!hasSignificantDigit) { + if (c == '0') { + leadingZeroes++; + } else { + hasSignificantDigit = true; + } + } continue; } else if (c == '.' && dot == -1) { dot = lo; @@ -178,7 +187,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p } if (!digitFound) { if (skippedZeroes) { + // The stripper only walks over zeroes, so the digit it gives back is a zero. digitLo--; + leadingZeroes = 1; } else { throw NumericException.instance() .put("invalid decimal: '").put(cs) @@ -196,9 +207,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p int literalScale = dot == -1 ? 0 : (digitHi - dot - 1); int literalDigits = digitHi - digitLo - (dot == -1 ? 0 : 1); - int virtualScale = literalScale; + long virtualScale = literalScale; - int exp = 0; + long exp = 0; if (lo != hi) { // Parses exponent if ((cs.charAt(lo) | 32) == 'e') { @@ -222,7 +233,7 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // If lossy is enabled, we can strip digits after the provided scale (as long as it is provided) if (lossy && scale >= 0 && virtualScale > scale) { - int drop = virtualScale - scale; + long drop = virtualScale - scale; while (drop > 0 && digitHi > digitLo) { digitHi--; if (digitHi == dot) { @@ -237,7 +248,12 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // We still need to adjust the exponent if the user specifies a target scale. if (scale == -1) { - scale = virtualScale; + if (virtualScale > decimal.getMaxScale()) { + throw NumericException.instance() + .put("decimal '").put(cs) + .put("' exceeds maximum allowed scale of ").put(decimal.getMaxScale()); + } + scale = (int) virtualScale; } else if (virtualScale <= scale) { exp += scale - virtualScale; } else { @@ -258,8 +274,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // precision, as long as it's lower. // Compute the final precision of the decimal - int pow = literalDigits + exp; - final int finalPrecision = Math.max(pow, scale + 1); + long pow = literalDigits + exp; + final long unscaledDigits = literalDigits > leadingZeroes ? pow - leadingZeroes : 0; + final long finalPrecision = Math.max(Math.max(unscaledDigits, scale), 1); if (precision != -1 && finalPrecision > precision) { throw NumericException.instance() .put("decimal '").put(cs) @@ -277,8 +294,14 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p if (p == dot) { continue; } - - decimal.addPowerOfTenMultiple(--pow, cs.charAt(p) - '0'); + int multiplier = cs.charAt(p) - '0'; + pow--; + if (multiplier != 0) { + if (pow < 0 || pow >= decimal.getMaxPrecision()) { + throw NumericException.instance().put("decimal '").put(cs).put("' exponent is out of range"); + } + decimal.addPowerOfTenMultiple((int) pow, multiplier); + } } decimal.setScale(scale); @@ -286,7 +309,7 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p decimal.negate(); } - return Numbers.encodeLowHighInts(finalPrecision, scale); + return Numbers.encodeLowHighInts((int) finalPrecision, scale); } /** @@ -315,9 +338,9 @@ private static boolean isDigit(char c) { * @return true if "NaN" or "Infinity" is found at position lo, false otherwise */ private static boolean isNanOrInfinite(char ch, CharSequence cs, int lo, int hi) { - return (ch == 'N' && lo + 2 < hi && cs.charAt(lo + 1) == 'a' && cs.charAt(lo + 2) == 'N') || - (ch == 'I' && lo + 7 < hi && cs.charAt(lo + 1) == 'n' && cs.charAt(lo + 2) == 'f' + return (ch == 'N' && hi - lo == 3 && cs.charAt(lo + 1) == 'a' && cs.charAt(lo + 2) == 'N') || + (ch == 'I' && hi - lo == 8 && cs.charAt(lo + 1) == 'n' && cs.charAt(lo + 2) == 'f' && cs.charAt(lo + 3) == 'i' && cs.charAt(lo + 4) == 'n' && cs.charAt(lo + 5) == 'i' && cs.charAt(lo + 6) == 't' && cs.charAt(lo + 7) == 'y'); } -} \ No newline at end of file +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java new file mode 100644 index 000000000..25a9df4b8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java @@ -0,0 +1,228 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; + +public class WebSocketClientSchemaNegotiationTest { + @Test + public void testDefaultDoesNotRequestAndIgnoresUnsolicitedConfirmation() throws Exception { + ScriptedSocket socket = new ScriptedSocket("X-QWP-Schema: enabled\r\n"); + try (TestClient client = new TestClient(socket)) { + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertFalse(socket.request().contains("X-QWP-Request-Schema:")); + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + + @Test + public void testRequestedConfirmationAndAbsentLegacyResponse() throws Exception { + ScriptedSocket socket = new ScriptedSocket("X-QWP-Schema: EnAbLeD\r\n", ""); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertTrue(socket.request().contains("X-QWP-Request-Schema: true\r\n")); + Assert.assertTrue(client.isQwpSchemaEnabled()); + + client.disconnect(); + Assert.assertFalse("disconnect must clear per-connection confirmation", client.isQwpSchemaEnabled()); + socket.nextResponse(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertTrue("the one-way request survives reuse", socket.request().contains("X-QWP-Request-Schema: true\r\n")); + Assert.assertFalse("an absent response must not retain stale state", client.isQwpSchemaEnabled()); + } + } + + @Test + public void testRejectsInvalidEmptyAndDuplicateConfirmations() throws Exception { + assertRejected("X-QWP-Schema: disabled\r\n", "Invalid X-QWP-Schema"); + assertRejected("X-QWP-Schema: \r\n", "Invalid X-QWP-Schema"); + assertRejected("X-QWP-Schema: enabled\r\nX-QWP-Schema: enabled\r\n", "Duplicate X-QWP-Schema"); + } + + @Test + public void testDoesNotAcceptPrefixedOrValueSpoofHeaders() throws Exception { + String[] headers = { + "Not-X-QWP-Schema: enabled\r\n", + "X-Comment: X-QWP-Schema: enabled\r\n" + }; + for (String header : headers) { + ScriptedSocket socket = new ScriptedSocket(header); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + } + + private static void assertRejected(String headers, String message) throws Exception { + ScriptedSocket socket = new ScriptedSocket(headers); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + try { + client.upgrade("/write/v4", 1000, null); + Assert.fail("expected invalid schema confirmation"); + } catch (HttpClientException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains(message)); + } + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + + private static final class ScriptedSocket implements Socket { + private final String[] extraHeaders; + private int responseIndex; + private byte[] response; + private int responsePosition; + private String request; + + private ScriptedSocket(String... extraHeaders) { + this.extraHeaders = extraHeaders; + } + + @Override + public void close() { + } + + @Override + public int getFd() { + return 1; + } + + @Override + public boolean isClosed() { + return false; + } + + void nextResponse() { + responseIndex++; + response = null; + responsePosition = 0; + } + + @Override + public void of(int fd) { + } + + String request() { + return request; + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + if (response == null || responsePosition == response.length) { + return 0; + } + int count = Math.min(bufferLen, response.length - responsePosition); + for (int i = 0; i < count; i++) { + Unsafe.getUnsafe().putByte(bufferPtr + i, response[responsePosition++]); + } + return count; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + byte[] bytes = new byte[bufferLen]; + for (int i = 0; i < bufferLen; i++) bytes[i] = Unsafe.getUnsafe().getByte(bufferPtr + i); + request = new String(bytes, StandardCharsets.US_ASCII); + String key = header(request, "Sec-WebSocket-Key:"); + response = response(key, extraHeaders[responseIndex]); + responsePosition = 0; + return bufferLen; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + + private static String header(String request, String name) { + int start = request.indexOf(name) + name.length(); + int end = request.indexOf("\r\n", start); + return request.substring(start, end).trim(); + } + + private static byte[] response(String key, String extraHeaders) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + String accept = Base64.getEncoder().encodeToString(sha1.digest( + (key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").getBytes(StandardCharsets.US_ASCII))); + return ("HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + accept + "\r\n" + + extraHeaders + + "\r\n").getBytes(StandardCharsets.US_ASCII); + } catch (Exception e) { + throw new AssertionError(e); + } + } + } + + private static final class TestClient extends WebSocketClient { + private TestClient(ScriptedSocket socket) { + super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java new file mode 100644 index 000000000..f49bde362 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java @@ -0,0 +1,259 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +public class QwpSchemaFeedbackResponseTest { + + @Test + public void testUpdatesAreAtomicAndEveryPrefixIsRejected() { + byte[] valid = updates("orders", "trades"); + WebSocketResponse response = new WebSocketResponse(); + withNative(valid, (ptr, length) -> { + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertEquals(2, response.getSchemaUpdateCount()); + Assert.assertEquals("orders", response.getSchemaUpdateTableName(0)); + Assert.assertEquals(0, response.getSchemaUpdate(0).getRequestId()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, response.getSchemaUpdate(0).getResult()); + for (int n = 0; n < length; n++) { + Assert.assertFalse("prefix " + n, response.readFrom(ptr, n, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.isSchemaInvalidation()); + } + }); + } + + @Test + public void testInvalidationAndFailedReadClearVisibleFeedback() { + WebSocketResponse response = new WebSocketResponse(); + byte[] invalidation = base((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)); + withNative(invalidation, (ptr, length) -> { + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertTrue(response.isSchemaInvalidation()); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.readFrom(ptr, length - 1, true)); + Assert.assertFalse(response.isSchemaInvalidation()); + Assert.assertFalse(response.hasSchemaUpdates()); + }); + } + + @Test + public void testNegotiationModesAndDurableAckRules() { + byte[] updates = updates("t"); + withNative(updates, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, false))); + byte[] reserved = base((byte) (WebSocketResponse.STATUS_OK | 0x40)); + withNative(reserved, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + byte[] durable = new byte[]{(byte) (WebSocketResponse.STATUS_DURABLE_ACK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL), 0, 0}; + withNative(durable, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testDuplicateNamesAreCaseInsensitive() { + byte[] duplicate = updates("foo", "FOO"); + withNative(duplicate, (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertFalse(response.readFrom(ptr, length, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + }); + } + + @Test + public void testNestedEnvelopeAndNonZeroFeedbackRequestIdRejected() { + byte[] nested = updatesWithPayload("t", new byte[]{'Q', 'W', 'P', '1', 1, 0, 0, 0, 0, 0, 0, 0}); + withNative(nested, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + byte[] payload = missingPayload(); + payload[1] = 1; + byte[] nonZero = updatesWithPayload("t", payload); + withNative(nonZero, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testInvalidSchemaColumnNamesRejectWholeAckAndNackWithoutPublishingPrefix() { + byte[] valid = missingPayload(); + byte[][] invalidSchemas = { + knownPayload("x", "x"), + knownPayload("x", "X"), + knownPayload("Ä", "ä"), + knownPayload("a/b"), + knownPayload("a\nb") + }; + for (byte[] invalid : invalidSchemas) { + for (byte status : new byte[]{WebSocketResponse.STATUS_OK, WebSocketResponse.STATUS_SCHEMA_MISMATCH}) { + byte[] feedback = updatesWithPayloads(status, new String[]{"first", "second"}, new byte[][]{valid, invalid}); + withNative(feedback, (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertFalse(response.readFrom(ptr, length, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.isSchemaInvalidation()); + }); + } + } + } + + @Test + public void testNackUpdatesPreserveBaseError() { + byte[] payload = missingPayload(); + byte[] name = "bad_table".getBytes(StandardCharsets.UTF_8); + byte[] message = "bad row".getBytes(StandardCharsets.UTF_8); + ByteBuffer b = ByteBuffer.allocate(11 + message.length + 2 + 2 + name.length + 4 + payload.length) + .order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_SCHEMA_MISMATCH | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + b.putLong(19).putShort((short) message.length).put(message).putShort((short) 1); + b.putShort((short) name.length).put(name).putInt(payload.length).put(payload); + withNative(b.array(), (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertEquals(WebSocketResponse.STATUS_SCHEMA_MISMATCH, response.getStatus()); + Assert.assertEquals(19, response.getSequence()); + Assert.assertEquals("bad row", response.getErrorMessage()); + Assert.assertEquals("bad_table", response.getSchemaUpdateTableName(0)); + }); + } + + @Test + public void testHostileCountLengthAndUtf8RejectedWithoutPublication() { + byte[] count = base((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + count = java.util.Arrays.copyOf(count, count.length + 2); + count[count.length - 2] = (byte) 0xff; + count[count.length - 1] = (byte) 0xff; + withNative(count, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + + byte[] invalidUtf8 = updates("t"); + invalidUtf8[15] = (byte) 0xc0; + withNative(invalidUtf8, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + + byte[] oversized = updates("t"); + ByteBuffer.wrap(oversized).order(ByteOrder.LITTLE_ENDIAN).putInt(16, -1); + withNative(oversized, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testOneMiBBoundAppliesOnlyToUpdates() { + int count = 3000; + String prefix = new String(new char[123]).replace('\0', '\u4e00'); + int nameLength = 373; + int length = 11 + count * (2 + nameLength + 8); + Assert.assertTrue(length > QwpSchemaProtocol.MAX_FRAME_SIZE); + byte[] base = new byte[length]; + ByteBuffer b = ByteBuffer.wrap(base).order(ByteOrder.LITTLE_ENDIAN); + b.put(WebSocketResponse.STATUS_OK).putLong(1).putShort((short) count); + for (int i = 0; i < count; i++) { + byte[] name = (prefix + String.format(java.util.Locale.ROOT, "%04d", i)).getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(nameLength, name.length); + b.putShort((short) nameLength).put(name).putLong(i); + } + withNative(base, (ptr, n) -> Assert.assertTrue(new WebSocketResponse().readFrom(ptr, n, false))); + base[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL); + withNative(base, (ptr, n) -> Assert.assertTrue(new WebSocketResponse().readFrom(ptr, n, true))); + byte[] payload = missingPayload(); + byte[] update = java.util.Arrays.copyOf(base, base.length + 2 + 2 + 1 + 4 + payload.length); + ByteBuffer trailer = ByteBuffer.wrap(update).order(ByteOrder.LITTLE_ENDIAN); + trailer.position(base.length).putShort((short) 1).putShort((short) 1).put((byte) 't').putInt(payload.length).put(payload); + update[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES); + withNative(update, (ptr, n) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, n, true))); + } + + private static byte[] base(byte status) { + ByteBuffer b = ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN); + b.put(status).putLong(7).putShort((short) 0); + return b.array(); + } + + private static byte[] missingPayload() { + return ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) QwpSchemaProtocol.RESULT_MISSING).array(); + } + + private static byte[] knownPayload(String... columnNames) { + int size = 26; + for (String name : columnNames) { + size += 2 + name.getBytes(StandardCharsets.UTF_8).length + 6; + } + ByteBuffer b = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) -1).putShort((short) columnNames.length); + for (String name : columnNames) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + b.putShort((short) bytes.length).put(bytes).putInt(5).putShort((short) 0); + } + return b.array(); + } + + private static byte[] updates(String... names) { + return updatesWithPayloads(names, missingPayload()); + } + + private static byte[] updatesWithPayload(String name, byte[] payload) { + return updatesWithPayloads(new String[]{name}, payload); + } + + private static byte[] updatesWithPayloads(String[] names, byte[] payload) { + byte[][] payloads = new byte[names.length][]; + java.util.Arrays.fill(payloads, payload); + return updatesWithPayloads(WebSocketResponse.STATUS_OK, names, payloads); + } + + private static byte[] updatesWithPayloads(byte status, String[] names, byte[][] payloads) { + int size = 13; + for (int i = 0; i < names.length; i++) { + size += 2 + names[i].getBytes(StandardCharsets.UTF_8).length + 4 + payloads[i].length; + } + ByteBuffer b = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (status | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + b.putLong(7).putShort((short) 0).putShort((short) names.length); + for (int i = 0; i < names.length; i++) { + String name = names[i]; + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + b.putShort((short) bytes.length).put(bytes).putInt(payloads[i].length).put(payloads[i]); + } + return b.array(); + } + + private static void withNative(byte[] bytes, NativeAssertion assertion) { + long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, bytes[i]); + } + assertion.run(ptr, bytes.length); + } finally { + Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private interface NativeAssertion { + void run(long ptr, int length); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java new file mode 100644 index 000000000..3335140f8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -0,0 +1,2332 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaSenderIntegrationTest { + private static final String DECIMAL_TEXT_CORPUS = + "/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv"; + + @Test + public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaSenderIntegrationTest.class.getResourceAsStream(DECIMAL_TEXT_CORPUS); + Assert.assertNotNull(DECIMAL_TEXT_CORPUS, stream); + boolean[][] sourceTargets = new boolean[3][2]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + DecimalTextVector vector = new DecimalTextVector(line.split("\t", -1)); + sourceTargets[vector.sourceIndex()][vector.targetIndex()] = true; + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 721 + count, 731 + count, vector.targetType()); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events"); + vector.append(sender, "value"); + if (vector.isNull()) { + sender.uuidColumn("failed_b", 11, 12); + } + sender.atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + if (vector.isNull()) { + reader.uuidOnlyTable("events", 721 + count, 731 + count, 11, 12); + } else { + reader.decimalTextTable( + "events", 721 + count, 731 + count, vector.expectedText); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(24, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target pair " + source + '/' + target, + sourceTargets[source][target]); + } + } + }); + } + + @Test + public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversRows() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(781, 791, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .decimalColumn("value", Decimal64.NULL_VALUE) + .decimalColumn("value", new Decimal64(12_340, 2)) + .binaryColumn("value", new byte[]{1}) + .atNow(); + + sender.decimalColumn("value", Decimal256.MAX_VALUE); + try { + sender.decimalColumn("failed_b", new Decimal64(1, 0)); + Assert.fail("expected unsupported DECIMAL64-to-UUID conversion"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.decimalColumn("value", new Decimal64(-125, 3)).atNow(); + sender.table("events").atNow(); + sender.decimalColumn("value", new Decimal256( + 0x161bcca7119915b5L, + 0x0764b4abe8652979L, + 0x7775a5f171950fffL, + 0xffffffffffffffffL, + 76)).atNow(); + sender.decimalColumn("value", new Decimal64(0, 0)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimalTextLifecycleTable("events", 781, 791); + } + }); + } + + @Test + public void testNativeDecimalTextMatchesServerScalarFormattingForRawPublicValues() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(801, 811, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + Decimal64 wrappedScale = new Decimal64(123, 1); + wrappedScale.setScale(257); + sender.table("events") + .decimalColumn("value", new Decimal128(Long.MAX_VALUE, -1L)) + .atNow(); + sender.decimalColumn("value", wrappedScale).atNow(); + sender.decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimalTextTable( + "events", + 801, + 811, + "170141183460469231731687303715884105727", + "12.3", + "4.20" + ); + } + }); + } + + @Test + public void testNativeDecimalRollbackAndTargetScaleWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(16, 4); + LongTextSchemaHandler handler = new LongTextSchemaHandler(601, 611, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(150, 2)).atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.decimalColumn("value", Decimal128.fromLong(100_000_000_000_000_000L, 0)); + Assert.fail("expected target precision overflow"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.decimalColumn("value", new Decimal64(25, 2)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).decimal64Table("events", 601, 611, 4, 15_000, 2_500); + } + }); + } + + @Test + public void testNarrowDecimalGenerationRemainsPinnedBeforeWideRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 621, 631, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", Decimal128.fromLong(1, 0)).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 0); + handler.version = 632; + sender.table("events"); + try { + sender.decimalColumn("value", Decimal128.fromLong(1_000, 0)); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.decimalColumn("value", Decimal128.fromLong(1_000, 0)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDecimalBlocks("events", 621, 631, 632); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testDecimalScaleGenerationRemainsPinnedBeforeScaleRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 641, 651, ColumnType.getDecimalType(18, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(1, 0)).atNow(); + handler.targetType = ColumnType.getDecimalType(18, 4); + handler.version = 652; + sender.table("events"); + try { + sender.decimalColumn("value", new Decimal64(12_345, 4)); + Assert.fail("expected scale-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.decimalColumn("value", new Decimal64(12_345, 4)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDecimal64ScaleBlocks( + "events", 641, 651, 652); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 661, 671, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 1).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 672; + sender.table("events"); + try { + sender.longColumn("value", 100_000_000_000_000L); + Assert.fail("expected width-and-scale-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.longColumn("value", 100_000_000_000_000L).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDecimalWidthAndScaleBlocks("events", 661, 671, 672); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 681, 691, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "1").atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 692; + sender.table("events"); + try { + sender.stringColumn("value", "100000000000000"); + Assert.fail("expected width-and-scale-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.stringColumn("value", "100000000000000").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDecimalWidthAndScaleBlocks("events", 681, 691, 692); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testFloatingDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 701, 711, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").floatColumn("value", 1.0f).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 712; + sender.table("events"); + try { + sender.doubleColumn("value", 100_000_000_000_000.0); + Assert.fail("expected width-and-scale-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.doubleColumn("value", 100_000_000_000_000.0).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDecimalWidthAndScaleBlocks("events", 701, 711, 712); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testNativeDecimalTextGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 741, 751, ColumnType.getDecimalType(5, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(12_340, 4)).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 752; + sender.table("events"); + try { + sender.decimalColumn("value", new Decimal64(250_000, 4)); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.decimalColumn("value", new Decimal64(250_000, 4)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDecimal64AndVarcharBlocks( + "events", 741, 751, 752); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToGeoHashNullWidthsRollbackAndRecovery() throws Exception { + assertMemoryLeak(() -> { + for (int bits : new int[]{1, 8, 60}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 471 + bits, 481, ColumnType.getGeoHashTypeWithBits(bits)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).nullGeoHashTable("events", 471 + bits, 481, bits); + } + } + + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 551, 561, ColumnType.getGeoHashTypeWithBits(8)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "zz").atNow(); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "a"); + Assert.fail("expected invalid GeoHash text"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "04").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).geoHashTable("events", 551, 561, 8, 0xff, 1); + } + }); + } + + @Test + public void testGeoHashPrecisionGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 571, 581, ColumnType.getGeoHashTypeWithBits(60)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "0123456789bc").atNow(); + handler.targetType = ColumnType.getGeoHashTypeWithBits(5); + handler.version = 582; + sender.table("events"); + try { + sender.stringColumn("value", "z"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.stringColumn("value", "z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoGeoHashBlocks("events", 571, 581, 582); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToLong256RollbackAndRecoveryWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(431, 441, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "0x0"); + Assert.fail("expected odd-length LONG256 rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "0x0123456789abcdeffedcba9876543210").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).long256Table("events", 431, 441); + } + }); + } + + @Test + public void testLong256GenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(451, 461, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .stringColumn("value", "0x0123456789abcdeffedcba9876543210") + .atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 462; + sender.table("events"); + try { + sender.stringColumn("value", "not-long256"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.stringColumn("value", "not-long256").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoLong256AndVarcharBlocks("events", 451, 461, 462); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToCharRollbackAndRecoveryWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(391, 401, ColumnType.CHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "A").atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.binaryColumn("value", new byte[]{1}); + Assert.fail("expected BINARY to CHAR rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.stringColumn("value", "\ud83d").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).charTable("events", 391, 401, 'A', '?'); + } + }); + } + + @Test + public void testVarcharGenerationRemainsPinnedBeforeNativeCharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(411, 421, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "native-a").atNow(); + handler.targetType = ColumnType.CHAR; + handler.version = 422; + sender.table("events"); + try { + sender.charColumn("value", 'B'); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.charColumn("value", '\uffff').atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoVarcharAndCharBlocks("events", 411, 421, 422); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testTimestampToTextTargetsRollbackAndRecoverWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(351 + target, 361, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.MICROS).atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); + Assert.fail("expected timestamp range error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.timestampColumn("value", 1, ChronoUnit.MILLIS).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).timestampTextTable( + "events", 351 + target, 361, "1970-01-01T00:00:00.001Z"); + } + } + }); + } + + @Test + public void testNativeTimestampGenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(371, 381, ColumnType.TIMESTAMP_NANO); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.NANOS).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 382; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 0); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.NANOS).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoTimestampAndVarcharBlocks( + "events", 371, 381, 382); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToTimestampTargetsUsePinnedWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(301 + target, 311, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "not-a-timestamp"); + Assert.fail("expected timestamp parse error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.stringColumn("value", "1969-12-31T23:59:59.999999Z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).longTimestampTable( + "events", + 301 + target, + 311, + target, + target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000 + ); + } + } + }); + } + + @Test + public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(321, 331, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "native-a").atNow(); + handler.targetType = ColumnType.TIMESTAMP_MICRO; + handler.version = 332; + sender.table("events"); + try { + sender.stringColumn("failed_b", "not-a-uuid"); + Assert.fail("expected invalid UUID and schema adoption"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "1970-01-01T00:00:00.000001Z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoVarcharAndTimestampBlocks( + "events", 321, 331, 332 + ); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongToTimestampTargetsUseRawUnitsAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", Long.MIN_VALUE); + sender.binaryColumn("value", new byte[]{1}).atNow(); + sender.table("events").longColumn("value", -1); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .longTimestampTable("events", 251 + target, 261, target, Long.MAX_VALUE); + Assert.assertEquals(2, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongGenerationRemainsNativeBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(271, 281, ColumnType.LONG); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 11).atNow(); + handler.targetType = ColumnType.TIMESTAMP_NANO; + handler.version = 282; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-long"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.longColumn("value", 12).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoLongAndTimestampBlocks("events", 271, 281, 282); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(231, 241, ColumnType.DOUBLE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleColumn("value", -0.0).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 242; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-double"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.doubleColumn("value", 1e23).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).twoDoubleAndVarcharBlocks("events", 231, 241, 242); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testFloatingToTextTargetsUsePinnedWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (String input : new String[]{"FLOAT", "DOUBLE"}) { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(211 + target, 221, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events"); + if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(0x7fc00002)); + else sender.doubleColumn("value", Double.longBitsToDouble(0x7ff8000000000002L)); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events"); + if ("FLOAT".equals(input)) sender.floatColumn("value", 0.1f); + else sender.doubleColumn("value", 1e23); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected floating row rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(1)); + else sender.doubleColumn("value", Double.longBitsToDouble(1)); + sender.atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).floatingTextTable( + "events", 211 + target, 221, target, + "FLOAT".equals(input) ? "0.10000000149011612" : "1.0E23", + "FLOAT".equals(input) ? "1.401298464324817E-45" : "5.0E-324" + ); + Assert.assertEquals(2, handler.describeRequests.get()); + } + } + } + }); + } + + @Test + public void testUuidGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(191, 201, ColumnType.UUID); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L) + .atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 202; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-uuid"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.uuidColumn("value", 0x2122232425262728L, 0x3132333435363738L).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).twoUuidAndVarcharBlocks("events", 191, 201, 202); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(171 + target, 181, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").uuidColumn("value", 7, 0); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L).atNow(); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.longTextTable( + "events", + 171 + target, + 181, + target, + "11121314-1516-1718-0102-030405060708" + ); + Assert.assertEquals(2, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(121 + target, 131, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("value", Long.MIN_VALUE); + // MIN is an effective null write, so this unsupported duplicate must be ignored. + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").longColumn("value", 7); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + // The failed middle row is auto-cancelled; continue without selecting the table again. + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.longTextTable("events", 121 + target, 131, target, Long.toString(Long.MAX_VALUE)); + Assert.assertEquals(2, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongSymbolGenerationRemainsPinnedAcrossTextRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(151, 161, ColumnType.SYMBOL); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("value", 7).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 162; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 2); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.longColumn("value", 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()) + .twoLongTextBlocks("events", 151, 161, 162); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testAsymmetricUuidRowsArePinnedAndInvalidMiddleRowRollsBack() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 41, 73); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("id", "11121314-1516-1718-0102-030405060708").atNow(); + sender.table("events"); + try { + sender.stringColumn("failed_b", "must-roll-back"); + sender.stringColumn("id", "not-a-uuid"); + Assert.fail("expected invalid UUID"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.uuidColumn("id", 0x2122232425262728L, 0x3132333435363738L).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + FrameReader reader = new FrameReader(frame); + reader.schemaTable("events", 41, 73, 2, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + Assert.assertEquals(0x2122232425262728L, reader.i64()); + Assert.assertEquals(0x3132333435363738L, reader.i64()); + reader.eof(); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testDuplicateSetterSkipsInvalidConversionAndRefresh() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 7, 9); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2); + sender.stringColumn("id", "invalid and ignored").atNow(); + sender.flush(); + Assert.assertNotNull(handler.awaitDataFrame()); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testExplicitAndServerAssignedDesignatedTimestampUseSchemaFrames() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 11, 12); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).at(Instant.ofEpochSecond(3, 4)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + } + }); + } + + @Test + public void testMissingTableInfersColumnsAndUsesCachedAbsence() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("missing").longColumn("x", 7).atNow(); + sender.table("missing").longColumn("x", 8).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("missing", -1, -1, 2, "x", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(8, reader.i64()); + reader.eof(); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testKnownAdoptionRetainsPendingUnknownGeneration() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, 31, 41); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 11).atNow(); + handler.result = QwpSchemaProtocol.RESULT_KNOWN; + try { + sender.stringColumn("failed_b", "partial"); + sender.stringColumn("id", "11121314-1516-1718-0102-030405060708"); + Assert.fail("expected schema adoption"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.stringColumn("id", "21222324-2526-2728-3132-333435363738").atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.twoInferredAdoptionBlocks("events"); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testUnchangedMissingColumnRefreshesOnceAndRetainsInferredRows() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 1).atNow(); + try { + sender.stringColumn("id", "different inferred type"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("id", 3).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", -1, -1, 2, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + reader.eof(); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testFreshLookupFailuresRemoveOnlyPartialInferredRow() throws Exception { + assertMemoryLeak(() -> { + int[] results = { + QwpSchemaProtocol.RESULT_DENIED, + QwpSchemaProtocol.RESULT_UNAVAILABLE, + QwpSchemaProtocol.RESULT_TOO_LARGE + }; + LineSenderSchemaException.Reason[] reasons = { + LineSenderSchemaException.Reason.ACCESS_DENIED, + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE + }; + for (int i = 0; i < results.length; i++) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 1).atNow(); + handler.result = results[i]; + try { + sender.stringColumn("failed_only", "partial"); + sender.stringColumn("id", "different inferred type"); + Assert.fail("expected fresh lookup failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reasons[i], e.getReason()); + } + handler.result = QwpSchemaProtocol.RESULT_MISSING; + sender.longColumn("id", 3).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", -1, -1, 2, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + reader.eof(); + Assert.assertEquals(3, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testNullNoopDoesNotTriggerNegotiation() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").doubleArray("values", (double[]) null); + sender.ipv4Column("address", (CharSequence) null); + Assert.assertEquals(0, handler.describeRequests.get()); + sender.cancelRow(); + } + }); + } + + @Test + public void testUnsupportedSetterCannotBypassSchemaAndInvalidNameDoesNotRefresh() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 2, 3); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events"); + try { + sender.longColumn("id", 1); + Assert.fail("expected UUID target rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals(1, handler.describeRequests.get()); + sender.table("events"); + try { + sender.stringColumn("bad\nname", "x"); + Assert.fail("expected invalid column name"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("column name")); + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyFrameAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + FrameReader reader = new FrameReader(frame); + reader.legacyVarcharTable("events", "value", "legacy"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 51, 52); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.version = 53; + sender.table("events"); + try { + sender.stringColumn("id", "still-invalid"); + Assert.fail("expected unchanged UUID validation error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.uuidColumn("id", 3, 4).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + new FrameReader(frame).twoUuidBlocks("events", 51, 52, 1, 2, 53, 3, 4); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testPendingLegacyRowSurvivesUpgradeAndPublishesBeforeSchemaRow() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler legacyHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + TestWebSocketServer legacy = new TestWebSocketServer(legacyHandler, false, null, port); + legacy.start(); + Assert.assertTrue(legacy.awaitStart(5, TimeUnit.SECONDS)); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = Sender.builder("ws::addr=localhost:" + port + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;") + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build(); + TestWebSocketServer upgraded = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + legacy.close(); + Assert.assertTrue("sender did not observe the legacy connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + sender.table("events").stringColumn("legacy_value", "B"); + SchemaHandler schemaHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 61, 62); + upgraded = new TestWebSocketServer(schemaHandler, false, null, port); + upgraded.setAdvertiseSchema(true); + upgraded.start(); + Assert.assertTrue(upgraded.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed and servicing frames", + schemaHandler.awaitPong()); + sender.atNow(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = schemaHandler.awaitDataFrames(2); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT) != 0); + Assert.assertTrue((frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("events", 61, 62, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, schemaHandler.describeRequests.get()); + } finally { + sender.close(); + legacy.close(); + if (upgraded != null) { + upgraded.close(); + } + } + }); + } + + @Test + public void testAsyncFirstEffectiveWriteWaitsForNegotiationInsteadOfGuessingLegacy() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + Future write = executor.submit(() -> { + entered.countDown(); + sender.table("events").uuidColumn("id", 7, 8).atNow(); + sender.flush(); + }); + TestWebSocketServer server = null; + try { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + try { + write.get(100, TimeUnit.MILLISECONDS); + Assert.fail("first effective write guessed a wire mode without a server handshake"); + } catch (TimeoutException expected) { + // The write remains blocked inside the bounded negotiation path. + } + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 71, 72); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try { + write.get(5, TimeUnit.SECONDS); + } catch (ExecutionException e) { + throw new AssertionError(e.getCause()); + } + byte[] frame = handler.awaitDataFrame(); + Assert.assertTrue((frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, handler.describeRequests.get()); + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + sender.close(); + if (server != null) { + server.close(); + } + } + }); + } + + @Test + public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 81, 82); + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + server.setAdvertisedMaxBatchSize(64); + try (TestWebSocketServer ignored = start(server); + Sender sender = sender(server)) { + sender.table("a").uuidColumn("id", 1, 2).atNow(); + sender.table("b").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = handler.awaitDataFrames(2); + for (int i = 0; i < 2; i++) { + byte[] frame = frames.get(i); + Assert.assertTrue(frame.length <= 64); + Assert.assertTrue((frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, + ByteBuffer.wrap(frame).order(ByteOrder.LITTLE_ENDIAN).getShort(6) & 0xffff); + } + Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT) != 0); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + FrameReader first = new FrameReader(frames.get(0)); + first.schemaTable("a", 81, 82, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, first.u8()); + Assert.assertEquals(1, first.i64()); + Assert.assertEquals(2, first.i64()); + first.eof(); + FrameReader second = new FrameReader(frames.get(1)); + second.schemaTable("b", 82, 82, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, second.u8()); + Assert.assertEquals(3, second.i64()); + Assert.assertEquals(4, second.i64()); + second.eof(); + } + }); + } + + @Test(timeout = 10_000) + public void testFirstNegotiationDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + sender.table("events"); + try { + sender.uuidColumn("id", 1, 2); + Assert.fail("expected schema negotiation deadline"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + try (TestWebSocketServer server = new TestWebSocketServer(handler, false, null, port)) { + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + sender.uuidColumn("id", 3, 4).atNow(); + sender.flush(); + Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + } + }); + } + + @Test + public void testInterruptedFirstNegotiationFailsTypedAndPreservesInterrupt() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;"); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + AtomicReference writer = new AtomicReference<>(); + AtomicBoolean interruptPreserved = new AtomicBoolean(); + Future result = executor.submit(() -> { + writer.set(Thread.currentThread()); + entered.countDown(); + try { + sender.table("events").uuidColumn("id", 1, 2); + return null; + } catch (LineSenderSchemaException e) { + interruptPreserved.set(Thread.currentThread().isInterrupted()); + return e; + } + }); + try { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + writer.get().interrupt(); + LineSenderSchemaException error = result.get(5, TimeUnit.SECONDS); + Assert.assertNotNull(error); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, error.getReason()); + Assert.assertTrue("writer interrupt status was cleared", interruptPreserved.get()); + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + sender.close(); + } + }); + } + + @Test + public void testResetDiscardsRetiredSchemaGeneration() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 101, 102); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.version = 103; + sender.table("events"); + try { + sender.stringColumn("id", "invalid"); + Assert.fail("expected UUID validation error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.reset(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", 101, 103, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(4, reader.i64()); + reader.eof(); + } + }); + } + + @Test + public void testNormalSenderAutomaticallyNegotiatesBeforeFirstEffectiveValue() throws Exception { + assertMemoryLeak(() -> { + DescribeUuidHandler handler = new DescribeUuidHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + sender.table("events"); + try { + sender.stringColumn("id", "not-a-uuid"); + sender.cancelRow(); + Assert.fail("expected schema-directed UUID validation"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + private static final class DescribeUuidHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger describeRequests = new AtomicInteger(); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + ByteBuffer input = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + if (input.getInt(0) != QwpConstants.MAGIC_MESSAGE + || data[5] != QwpSchemaProtocol.FLAG_CONTROL) { + return; + } + describeRequests.incrementAndGet(); + long requestId = input.getLong(QwpConstants.HEADER_SIZE + 1); + byte[] column = "id".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + ByteBuffer response = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + response.putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(17) + .putLong(23) + .putShort((short) -1) + .putShort((short) 1) + .putShort((short) column.length) + .put(column) + .putInt(ColumnType.UUID) + .putShort((short) 0); + try { + client.sendBinary(response.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private static TestWebSocketServer legacyServer(SchemaHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + return start(server); + } + + private static TestWebSocketServer start(TestWebSocketServer server) throws Exception { + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + private static Sender sender(TestWebSocketServer server) { + return Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); + } + + private static final class DecimalTextVector { + private final String caseId; + private final String expectedText; + private final String inputForm; + private final String sourceType; + private final int sourceScale; + private final String[] limbs; + private final String targetType; + + private DecimalTextVector(String[] fields) { + Assert.assertEquals(String.join("\t", fields), 10, fields.length); + caseId = fields[0]; + sourceType = fields[1]; + sourceScale = Integer.parseInt(fields[2]); + limbs = new String[]{fields[3], fields[4], fields[5], fields[6]}; + targetType = fields[7]; + inputForm = fields[8]; + expectedText = fields[9]; + } + + private void append(Sender sender, String column) { + if ("JAVA_NULL".equals(inputForm)) { + appendJavaNull(sender, column); + return; + } + if ("NULL_VALUE".equals(inputForm)) { + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, Decimal64.NULL_VALUE); + return; + case "DECIMAL128": + sender.decimalColumn(column, Decimal128.NULL_VALUE); + return; + case "DECIMAL256": + sender.decimalColumn(column, Decimal256.NULL_VALUE); + return; + default: + throw new AssertionError(sourceType); + } + } + Assert.assertEquals("VALUE", inputForm); + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, new Decimal64(limb(0), sourceScale)); + return; + case "DECIMAL128": + sender.decimalColumn(column, new Decimal128(limb(1), limb(0), sourceScale)); + return; + case "DECIMAL256": + sender.decimalColumn(column, + new Decimal256(limb(3), limb(2), limb(1), limb(0), sourceScale)); + return; + default: + throw new AssertionError(sourceType); + } + } + + private void appendJavaNull(Sender sender, String column) { + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, (Decimal64) null); + return; + case "DECIMAL128": + sender.decimalColumn(column, (Decimal128) null); + return; + case "DECIMAL256": + sender.decimalColumn(column, (Decimal256) null); + return; + default: + throw new AssertionError(sourceType); + } + } + + private boolean isNull() { + return !"VALUE".equals(inputForm); + } + + private long limb(int index) { + Assert.assertNotEquals("-", limbs[index]); + return Long.parseUnsignedLong(limbs[index], 16); + } + + private int sourceIndex() { + switch (sourceType) { + case "DECIMAL64": + return 0; + case "DECIMAL128": + return 1; + case "DECIMAL256": + return 2; + default: + throw new AssertionError(sourceType); + } + } + + private int targetIndex() { + if ("STRING".equals(targetType)) { + return 0; + } + if ("VARCHAR".equals(targetType)) { + return 1; + } + throw new AssertionError(targetType); + } + + private int targetType() { + if ("STRING".equals(targetType)) { + return ColumnType.STRING; + } + if ("VARCHAR".equals(targetType)) { + return ColumnType.VARCHAR; + } + throw new AssertionError(targetType); + } + } + + private static final class FrameReader { + private final ByteBuffer in; + + private FrameReader(byte[] data) { + in = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + } + + private long i64() { + return in.getLong(); + } + + private void legacyVarcharTable(String table, String column, String... values) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(values.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + int total = 0; + StringBuilder joined = new StringBuilder(); + for (String value : values) { + joined.append(value); + total += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(total, in.getInt()); + } + byte[] bytes = joined.toString().getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[total]; + in.get(actual); + Assert.assertArrayEquals(bytes, actual); + eof(); + } + + private void longTextTable( + String table, + int tableId, + long version, + int targetType, + String expectedValue + ) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + // Global dictionary registration is append-only: the rolled-back B value + // remains an unused dictionary entry, while no B row survives. + Assert.assertEquals("7", string()); + Assert.assertEquals(expectedValue, string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, tableId, version, 2, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(1, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + } + eof(); + } + + private void long256Table(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0xfedcba9876543210L, i64()); + Assert.assertEquals(0x0123456789abcdefL, i64()); + Assert.assertEquals(0, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void geoHashTable( + String table, int tableId, long version, int bits, int first, int second + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(bits, varint()); + Assert.assertEquals(first, u8()); + Assert.assertEquals(second, u8()); + eof(); + } + + private void decimal64Table( + String table, int tableId, long version, int scale, long first, long second + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(scale, u8()); + Assert.assertEquals(first, i64()); + Assert.assertEquals(second, i64()); + eof(); + } + + private void decimalTextLifecycleTable(String table, int tableId, long version) { + StringBuilder maxScale = new StringBuilder(78).append("0."); + for (int i = 0; i < 76; i++) { + maxScale.append('9'); + } + String expected = "123.40-0.125" + maxScale + '0'; + + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 5, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0x04, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals(12, in.getInt()); + Assert.assertEquals(90, in.getInt()); + Assert.assertEquals(91, in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void decimalTextTable(String table, int tableId, long version, String... expectedValues) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, expectedValues.length, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + int totalLength = 0; + StringBuilder expected = new StringBuilder(); + for (String value : expectedValues) { + expected.append(value); + totalLength += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(totalLength, in.getInt()); + } + byte[] expectedBytes = expected.toString().getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[totalLength]; + in.get(actual); + Assert.assertArrayEquals(expectedBytes, actual); + eof(); + } + + private void twoDecimalBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1_000, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void twoDecimal64ScaleBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(12_345, i64()); + eof(); + } + + private void twoDecimal64AndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(12_340, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(7, in.getInt()); + Assert.assertEquals("25.0000", stringBytes(7)); + eof(); + } + + private void twoDecimalWidthAndScaleBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(10_000, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, u8()); + Assert.assertEquals(-8_446_744_073_709_551_616L, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void nullGeoHashTable(String table, int tableId, long version, int bits) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(bits, varint()); + eof(); + } + + private void twoGeoHashBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(60, varint()); + Assert.assertEquals(0x443214c74254bL, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, varint()); + Assert.assertEquals(31, u8()); + eof(); + } + + private void twoLong256AndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0xfedcba9876543210L, i64()); + Assert.assertEquals(0x0123456789abcdefL, i64()); + Assert.assertEquals(0, i64()); + Assert.assertEquals(0, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(11, in.getInt()); + Assert.assertEquals("not-long256", stringBytes(11)); + eof(); + } + + private void longTimestampTable( + String table, int tableId, long version, int targetType, long expectedValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", + targetType == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedValue, i64()); + eof(); + } + + private void twoLongAndTimestampBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(12, i64()); + eof(); + } + + private void twoVarcharAndTimestampBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + eof(); + } + + private void timestampTextTable( + String table, int tableId, long version, String expectedLastValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedLastValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + eof(); + } + + private void charTable(String table, int tableId, long version, char first, char second) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_CHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(first, in.getChar()); + Assert.assertEquals(second, in.getChar()); + eof(); + } + + private void twoVarcharAndCharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_CHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0xffff, in.getChar()); + eof(); + } + + private void twoTimestampAndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Long.MIN_VALUE, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + String expected = "2262-04-11T23:47:16.854Z"; + byte[] bytes = expected.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(bytes.length, in.getInt()); + byte[] actual = new byte[bytes.length]; + in.get(actual); + Assert.assertArrayEquals(bytes, actual); + eof(); + } + + private void floatingTextTable( + String table, + int tableId, + long version, + int targetType, + String rolledBackValue, + String expectedValue + ) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(rolledBackValue, string()); + Assert.assertEquals(expectedValue, string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, tableId, version, 2, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(1, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + } + eof(); + } + + private void messageHeader(int tableCount) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(tableCount, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + } + + private void schemaBlockHeader( + String table, + int tableId, + long version, + int rows, + String column, + byte wireType + ) { + Assert.assertEquals(table, string()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(tableId, in.getInt()); + Assert.assertEquals(version, in.getLong()); + Assert.assertEquals(rows, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(wireType & 0xff, u8()); + } + + private void twoLongTextBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("7", string()); + + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_SYMBOL); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(1, in.getInt()); + Assert.assertEquals('8', u8()); + eof(); + } + + private void twoUuidAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x0102030405060708L, i64()); + Assert.assertEquals(0x1112131415161718L, i64()); + + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(36, in.getInt()); + Assert.assertEquals("31323334-3536-3738-2122-232425262728", stringBytes(36)); + eof(); + } + + private void twoDoubleAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Double.doubleToRawLongBits(-0.0), i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals("1.0E23", stringBytes(6)); + eof(); + } + + private String stringBytes(int length) { + byte[] bytes = new byte[length]; + in.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private void eof() { + Assert.assertFalse("unexpected trailing column or value bytes", in.hasRemaining()); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + in.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private void schemaTable(String table, int tableId, long version, int rows, String column, byte type) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + if (tableId < 0) { + Assert.assertEquals(0, u8()); + } else { + Assert.assertEquals(1, u8()); + Assert.assertEquals(tableId, in.getInt()); + Assert.assertEquals(version, in.getLong()); + } + Assert.assertEquals(rows, varint()); + int columns = varint(); + Assert.assertEquals(1, columns); + Assert.assertEquals(column, string()); + Assert.assertEquals(type & 0xff, u8()); + } + + private void twoUuidBlocks( + String table, + int tableId, + long firstVersion, + long firstLo, + long firstHi, + long secondVersion, + long secondLo, + long secondHi + ) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(2, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + uuidBlock(table, tableId, firstVersion, firstLo, firstHi); + uuidBlock(table, tableId, secondVersion, secondLo, secondHi); + eof(); + } + + private void twoInferredAdoptionBlocks(String table) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(2, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + Assert.assertEquals(table, string()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("id", string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + + uuidBlock(table, 31, 41, 0x3132333435363738L, 0x2122232425262728L); + eof(); + } + + private void uuidBlock(String table, int tableId, long version, long lo, long hi) { + Assert.assertEquals(table, string()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(tableId, in.getInt()); + Assert.assertEquals(version, in.getLong()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("id", string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(lo, i64()); + Assert.assertEquals(hi, i64()); + } + + private void uuidOnlyTable(String table, int tableId, long version, long lo, long hi) { + schemaTable(table, tableId, version, 1, "failed_b", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(lo, i64()); + Assert.assertEquals(hi, i64()); + eof(); + } + + private int u8() { + return in.get() & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int b; + do { + b = u8(); + result |= (b & 0x7f) << shift; + shift += 7; + } while ((b & 0x80) != 0); + return result; + } + } + + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final List dataFrames = new ArrayList<>(); + private final CountDownLatch pong = new CountDownLatch(1); + private final AtomicInteger describeRequests = new AtomicInteger(); + private volatile int result; + private final int tableId; + private volatile long version; + private long ackSequence; + + private SchemaHandler(int result, int tableId, long version) { + this.result = result; + this.tableId = tableId; + this.version = version; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + describeRequests.incrementAndGet(); + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); + int nameLength = request.getShort(QwpConstants.HEADER_SIZE + 1 + Long.BYTES) & 0xffff; + byte[] name = new byte[nameLength]; + request.position(QwpConstants.HEADER_SIZE + 1 + Long.BYTES + Short.BYTES); + request.get(name); + int responseTableId = tableId + ("b".equals(new String(name, StandardCharsets.UTF_8)) ? 1 : 0); + sendSchema(client, requestId, responseTableId); + return; + } + dataFrames.add(data); + notifyAll(); + try { + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence++)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onOpen(TestWebSocketServer.ClientHandler client) { + try { + client.sendPing(new byte[]{0x51, 0x57, 0x50}); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onPong(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length == 3 && data[0] == 0x51 && data[1] == 0x57 && data[2] == 0x50) { + pong.countDown(); + } + } + + private boolean awaitPong() throws InterruptedException { + return pong.await(5, TimeUnit.SECONDS); + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private synchronized List awaitDataFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.size() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for " + count + " data frames; received " + dataFrames.size()); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return new ArrayList<>(dataFrames); + } + + private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId, int responseTableId) { + byte[] id = "id".getBytes(StandardCharsets.UTF_8); + byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); + byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1; + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + payloadLength += 4 + 8 + 2 + 2 + + 2 + id.length + 4 + 2 + + 2 + failed.length + 4 + 2 + + 2 + ts.length + 4 + 2; + } + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + out.putInt(responseTableId).putLong(version).putShort((short) 2).putShort((short) 3) + .putShort((short) id.length).put(id).putInt(ColumnType.UUID).putShort((short) 0) + .putShort((short) failed.length).put(failed).putInt(ColumnType.STRING).putShort((short) 0) + .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); + } + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private static final class LongTextSchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final List dataFrames = new ArrayList<>(); + private final AtomicInteger describeRequests = new AtomicInteger(); + private final int tableId; + private volatile int targetType; + private volatile long version; + private long ackSequence; + + private LongTextSchemaHandler(int tableId, long version, int targetType) { + this.tableId = tableId; + this.version = version; + this.targetType = targetType; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + describeRequests.incrementAndGet(); + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); + sendSchema(client, requestId); + return; + } + dataFrames.add(data); + notifyAll(); + try { + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence++)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for LONG-to-text data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId) { + byte[] value = "value".getBytes(StandardCharsets.UTF_8); + byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); + byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + + 2 + value.length + 4 + 2 + + 2 + failed.length + 4 + 2 + + 2 + ts.length + 4 + 2; + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(tableId).putLong(version).putShort((short) 2).putShort((short) 3) + .putShort((short) value.length).put(value).putInt(targetType).putShort((short) 0) + .putShort((short) failed.length).put(failed).putInt(ColumnType.UUID).putShort((short) 0) + .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 44f711a3d..a4c6dd062 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -43,6 +43,153 @@ */ public class QwpWebSocketEncoderTest { + @Test + public void testHeaderRejectsTableCountOutsideUnsignedShortRange() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.writeHeader(0, 0); + encoder.writeHeader(0xffff, 0); + assertIllegalArgument(() -> encoder.writeHeader(-1, 0)); + assertIllegalArgument(() -> encoder.writeHeader(0x1_0000, 0)); + } + }); + } + + private static void assertIllegalArgument(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertIllegalState(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + // expected + } + } + + @Test + public void testEncodeSchemaKnownAndUnknownIdentityWire() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + buffer.getOrCreateColumn("x", TYPE_LONG, true).addLong(7); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer, 42, 19); + long p = encoder.getBuffer().getBufferPtr(); + Assert.assertEquals(FLAG_GORILLA | FLAG_SCHEMA, Unsafe.getUnsafe().getByte(p + HEADER_OFFSET_FLAGS)); + Assert.assertEquals(1, Unsafe.getUnsafe().getByte(p + 14)); + Assert.assertEquals(42, Unsafe.getUnsafe().getInt(p + 15)); + Assert.assertEquals(19, Unsafe.getUnsafe().getLong(p + 19)); + Assert.assertEquals(size - HEADER_SIZE, Unsafe.getUnsafe().getInt(p + 8)); + + size = encoder.encodeSchema(buffer, -1, -1); + p = encoder.getBuffer().getBufferPtr(); + Assert.assertEquals(0, Unsafe.getUnsafe().getByte(p + 14)); + Assert.assertEquals(1, Unsafe.getUnsafe().getByte(p + 15)); + Assert.assertEquals(size - HEADER_SIZE, Unsafe.getUnsafe().getInt(p + 8)); + } + }); + } + + @Test + public void testSchemaMessageRejectsMixedModesAndInvalidIdentity() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertIllegalArgument(() -> encoder.encodeSchema(buffer, -1, 1)); + assertIllegalArgument(() -> encoder.encodeSchema(buffer, 0, -1)); + assertIllegalArgument(() -> encoder.beginSchemaMessage(0, new GlobalSymbolDictionary(), -1, -1)); + encoder.beginSchemaMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertIllegalState(() -> encoder.addTable(buffer)); + encoder.beginMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertIllegalState(() -> encoder.addSchemaTable(buffer, 1, 1)); + } + }); + } + + @Test + public void testSchemaMultiTableAndLegacyReuseHaveExactFlags() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer first = new QwpTableBuffer("a"); + QwpTableBuffer second = new QwpTableBuffer("b")) { + first.getOrCreateColumn("x", TYPE_LONG, true).addLong(1); + first.nextRow(); + second.getOrCreateColumn("x", TYPE_LONG, true).addLong(2); + second.nextRow(); + int legacySize = encoder.encode(first); + byte[] expectedLegacy = new byte[legacySize]; + for (int i = 0; i < legacySize; i++) { + expectedLegacy[i] = Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + i); + } + encoder.beginSchemaMessage(2, new GlobalSymbolDictionary(), -1, -1); + encoder.addSchemaTable(first, 7, 8); + encoder.addSchemaTable(second, -1, -1); + int size = encoder.finishMessage(); + long p = encoder.getBuffer().getBufferPtr(); + Assert.assertEquals(FLAG_GORILLA | FLAG_DELTA_SYMBOL_DICT | FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(p + HEADER_OFFSET_FLAGS)); + Assert.assertEquals(2, Unsafe.getUnsafe().getShort(p + 6)); + Assert.assertEquals(size - HEADER_SIZE, Unsafe.getUnsafe().getInt(p + 8)); + + Assert.assertEquals(legacySize, encoder.encode(first)); + Assert.assertEquals(FLAG_GORILLA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + HEADER_OFFSET_FLAGS)); + for (int i = 0; i < legacySize; i++) { + Assert.assertEquals("legacy byte " + i, expectedLegacy[i], + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + i)); + } + } + }); + } + + @Test + public void testSchemaSplitPreservesIdentityBodyAndPerSplitDeferFlag() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + MicrobatchBuffer firstSplit = new MicrobatchBuffer(64); + MicrobatchBuffer secondSplit = new MicrobatchBuffer(64); + QwpTableBuffer first = new QwpTableBuffer("a"); + QwpTableBuffer second = new QwpTableBuffer("b")) { + first.getOrCreateColumn("x", TYPE_LONG, true).addLong(11); + first.nextRow(); + second.getOrCreateColumn("x", TYPE_LONG, true).addLong(22); + second.nextRow(); + encoder.beginSchemaMessage(2, new GlobalSymbolDictionary(), -1, -1); + int firstOffset = encoder.getBuffer().getPosition(); + encoder.addSchemaTable(first, 7, 8); + int firstLength = encoder.getBuffer().getPosition() - firstOffset; + int secondOffset = encoder.getBuffer().getPosition(); + encoder.addSchemaTable(second, 9, 10); + int secondLength = encoder.getBuffer().getPosition() - secondOffset; + encoder.finishMessage(); + + encoder.copySplitMessage(firstSplit, firstOffset, firstLength, true, -1, -1); + encoder.copySplitMessage(secondSplit, secondOffset, secondLength, false, -1, -1); + Assert.assertEquals(FLAG_GORILLA | FLAG_DELTA_SYMBOL_DICT | FLAG_SCHEMA | FLAG_DEFER_COMMIT, + Unsafe.getUnsafe().getByte(firstSplit.getBufferPtr() + HEADER_OFFSET_FLAGS)); + Assert.assertEquals(FLAG_GORILLA | FLAG_DELTA_SYMBOL_DICT | FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(secondSplit.getBufferPtr() + HEADER_OFFSET_FLAGS)); + assertSliceEquals(encoder.getBuffer().getBufferPtr() + firstOffset, firstSplit, firstLength); + assertSliceEquals(encoder.getBuffer().getBufferPtr() + secondOffset, secondSplit, secondLength); + } + }); + } + + private static void assertSliceEquals(long expected, MicrobatchBuffer split, int length) { + long actual = split.getBufferPtr() + HEADER_SIZE + 2; // empty delta start/count varints + for (int i = 0; i < length; i++) { + Assert.assertEquals("table body byte " + i, + Unsafe.getUnsafe().getByte(expected + i), Unsafe.getUnsafe().getByte(actual + i)); + } + } + @Test public void testBufferResetAndReuse() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 3911f5c05..cd598fa70 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -44,6 +44,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -185,7 +186,8 @@ public void testBinaryColumnAfterCloseThrows() throws Exception { @Test public void testBinaryColumnRejectsNullArray() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.binaryColumn("x", (byte[]) null); @@ -200,7 +202,8 @@ public void testBinaryColumnRejectsNullArray() throws Exception { @Test public void testBinaryColumnRejectsNullDirectByteSlice() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.binaryColumn("x", (io.questdb.client.std.bytes.DirectByteSlice) null); @@ -257,7 +260,8 @@ public void testCancelRowAfterCloseThrows() throws Exception { @Test public void testCancelRowDiscardsPartialRow() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("test"); sender.longColumn("x", 1); sender.boolColumn("y", true); @@ -430,7 +434,8 @@ public void testIpv4ColumnStringRejectsNullLiteral() throws Exception { // real IPv4". Reject it as an invalid IPv4 address; the user can // pass a null reference (or omit the setter) to mark the row null. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); for (String input : new String[]{"null", "NULL", "Null", "nUlL"}) { try { @@ -456,7 +461,8 @@ public void testIpv4ColumnStringRejectsZeroAddress() throws Exception { // valid dotted-quad. Reject it on the client so the user gets a // clear error instead of a value that disappears on read. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.ipv4Column("addr", "0.0.0.0"); @@ -560,15 +566,8 @@ public void testOperationsAfterCloseThrow() throws Exception { @Test public void testPendingBytesMatchesGroundTruthAcrossTableSwitches() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - // Bypass ensureConnected: sendRow only short-circuits on - // connected==true and we don't drive any path that touches - // the cursor engine in this test. - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { // Round-robin three tables to exercise the per-row delta // logic across switches. The running pendingBytes must @@ -590,12 +589,8 @@ public void testPendingBytesMatchesGroundTruthAcrossTableSwitches() throws Excep @Test public void testPendingBytesUnchangedByCancelRow() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t0").longColumn("a", 1).longColumn("b", 2).atNow(); long committed = sender.getPendingBytes(); @@ -636,12 +631,8 @@ public void testRowAcceptedWhenServerAdvertisesNoCap() throws Exception { // the volatile serverMaxBatchSize to 0 mid-row, and a torn read that observed // the drop between the guard and the throw must not spuriously reject the row. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(0); Assert.assertEquals(0, sender.getServerMaxBatchSize()); @@ -665,12 +656,8 @@ public void testRowExceedingServerBatchCapThrows() throws Exception { // already exceed the server's advertised cap, before nextRow() commits it, so // the flush cannot build an oversize WS frame the server closes with 1009. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(64); Assert.assertEquals(64, sender.getServerMaxBatchSize()); @@ -706,12 +693,8 @@ public void testCumulativeBufferedBytesDoNotTripPerRowCap() throws Exception { // row once the table's CUMULATIVE buffered bytes crossed the cap; this // pins the wiring the single-row tests cannot distinguish. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(64); // Three rows of 22-26 bytes each: every row fits the 64-byte cap, @@ -721,6 +704,9 @@ public void testCumulativeBufferedBytesDoNotTripPerRowCap() throws Exception { } QwpTableBuffer buf = sender.getTableBuffer("t"); Assert.assertEquals(3, buf.getRowCount()); + // The accumulated frame is intentionally larger than the cap; + // this test covers the per-row guard, not close-time flushing. + sender.reset(); } }); } @@ -863,6 +849,49 @@ private static MicrobatchBuffer getMicrobatchBuffer(QwpWebSocketSender sender, S return (MicrobatchBuffer) field.get(sender); } + private static QwpWebSocketSender connectLegacySender( + TestWebSocketServer server, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos + ) { + CursorSendEngine engine = new CursorSendEngine(null, 4L * 1024 * 1024); + try { + return QwpWebSocketSender.connect( + "localhost", server.getPort(), null, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + null, false, engine, 0L); + } catch (Throwable t) { + engine.close(); + throw t; + } + } + + private static TestWebSocketServer startLegacyServer() throws Exception { + AtomicLong nextSequence = new AtomicLong(); + TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(QwpWireTestUtils.buildAck(nextSequence.getAndIncrement())); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + boolean started = false; + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + started = true; + return server; + } finally { + if (!started) { + server.close(); + } + } + } + /** * Creates a sender without connecting. * For unit tests that don't need actual connectivity. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java index b70bdc4f7..299bba248 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java @@ -400,7 +400,8 @@ public void testUnknownStatusByte() throws Exception { int size = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE; long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); try { - Unsafe.getUnsafe().putByte(ptr, (byte) 0xFF); + // Unknown low-six-bit status remains a legacy error response. + Unsafe.getUnsafe().putByte(ptr, (byte) 0x3F); Unsafe.getUnsafe().putLong(ptr + 1, 1L); Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); @@ -409,8 +410,8 @@ public void testUnknownStatusByte() throws Exception { WebSocketResponse parsed = new WebSocketResponse(); Assert.assertTrue(parsed.readFrom(ptr, size)); - Assert.assertEquals((byte) 0xFF, parsed.getStatus()); - Assert.assertEquals("UNKNOWN(255)", parsed.getStatusName()); + Assert.assertEquals((byte) 0x3F, parsed.getStatus()); + Assert.assertEquals("UNKNOWN(63)", parsed.getStatusName()); } finally { Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java new file mode 100644 index 000000000..0935943b6 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java @@ -0,0 +1,408 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class CursorWebSocketSchemaCoordinatorTest { + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testColdHitCaseAliasAndForcedRefreshUseProductionTransport() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("cold").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + try { + loop.resolveSchema(new String(new char[QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + 1]), 5_000); + Assert.fail("expected invalid table name"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, handler.requests.get()); + } + QwpSchemaResponse first = loop.resolveSchema("Trades", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, first.getResult()); + Assert.assertEquals(1, handler.requests.get()); + Assert.assertEquals(1, loop.resolveSchema("trades", 5_000).getTableId()); + Assert.assertEquals(1, handler.requests.get()); + Assert.assertEquals(2, loop.refreshSchema("TRADES", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testMissingIsCachedButRemoteFailuresAreTyped() throws Exception { + DescribeHandler missing = new DescribeHandler(true, QwpSchemaProtocol.RESULT_MISSING); + try (TestWebSocketServer server = server(missing); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("missing").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("Absent", 5_000).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("ABSENT", 0).getResult()); + Assert.assertEquals(1, missing.requests.get()); + } finally { + loop.close(); + } + } + assertRemoteReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "denied"); + assertRemoteReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "unavailable"); + assertRemoteReason(QwpSchemaProtocol.RESULT_TOO_LARGE, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "too-large"); + } + + @Test + public void testOrderedFeedbackInvalidationAndDurableDemultiplexing() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler, true); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("feedback").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort(), true, true)) { + Assert.assertTrue(client.isServerDurableAckEnabled()); + CursorWebSocketSendLoop loop = loop(client, engine, true); + try { + loop.start(); + loop.resolveSchema("seed", 5_000); + + handler.sendFeedback("Feedback", QwpSchemaProtocol.RESULT_MISSING); + handler.durableBeforeReply = true; + loop.refreshSchema("marker", 5_000); + int afterMarker = handler.requests.get(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("feedback", 0).getResult()); + Assert.assertEquals(afterMarker, handler.requests.get()); + + handler.sendInvalidation(); + loop.refreshSchema("marker", 5_000); + loop.resolveSchema("feedback", 5_000); + Assert.assertEquals(afterMarker + 2, handler.requests.get()); + Assert.assertEquals(-1, engine.ackedFsn()); + Assert.assertEquals(-1, engine.publishedFsn()); + } finally { + loop.close(); + } + } + } + + @Test + public void testFailedRefreshEvictsPriorCachedSchema() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("refresh-eviction").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertEquals(1, loop.resolveSchema("cached", 5_000).getTableId()); + handler.reply = false; + Future refresh = executor.submit(() -> loop.refreshSchema("CACHED", 200)); + Assert.assertTrue(handler.awaitRequests(2)); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, refresh); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("cached", 0)); + handler.reply = true; + Assert.assertEquals(3, loop.resolveSchema("cached", 5_000).getTableId()); + Assert.assertEquals(3, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testTimeoutIgnoresLateReplyAndNextRequestSucceeds() throws Exception { + DescribeHandler handler = new DescribeHandler(false); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("late").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Future timedOut = executor.submit(() -> loop.resolveSchema("late", 200)); + Assert.assertTrue(handler.awaitFirstRequest()); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, timedOut); + handler.replyToFirst(); + handler.reply = true; + Assert.assertEquals(2, loop.resolveSchema("late", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testUnnegotiatedConnectionIsUnsupported() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = new TestWebSocketServer(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("legacy").getAbsolutePath(), 1 << 20)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (WebSocketClient client = connect(server.getPort(), false)) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(0, handler.requests.get()); + } finally { + loop.close(); + } + } + } + } + + @Test + public void testMalformedControlFailsLookupWithoutAckFallback() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("malformed-control").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(1, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + private static CursorWebSocketSendLoop loop(WebSocketClient client, CursorSendEngine engine) { + return loop(client, engine, false); + } + + private static CursorWebSocketSendLoop loop(WebSocketClient client, CursorSendEngine engine, boolean durable) { + return new CursorWebSocketSendLoop(client, engine, 0, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, null, 1, 4, durable); + } + + private void assertRemoteReason(int result, LineSenderSchemaException.Reason reason, String directory) throws Exception { + DescribeHandler handler = new DescribeHandler(true, result); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder(directory).getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(reason, () -> loop.resolveSchema("t", 5_000)); + } finally { + loop.close(); + } + } + } + + private static TestWebSocketServer server(DescribeHandler handler) throws Exception { + return server(handler, false); + } + + private static TestWebSocketServer server(DescribeHandler handler, boolean durable) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler, durable); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static WebSocketClient connect(int port) { + return connect(port, true); + } + + private static WebSocketClient connect(int port, boolean schema) { + return connect(port, schema, false); + } + + private static WebSocketClient connect(int port, boolean schema, boolean durable) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + client.connect("localhost", port); + if (schema) { + client.requestQwpSchema(); + } + client.setQwpRequestDurableAck(durable); + client.upgrade("/write/v4", 5_000, null); + return client; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + try { + action.run(); + Assert.fail("expected schema failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + } + } + + private static void assertFutureReason( + LineSenderSchemaException.Reason reason, + Future future + ) throws Exception { + try { + future.get(5, TimeUnit.SECONDS); + Assert.fail("expected schema failure"); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof LineSenderSchemaException); + Assert.assertEquals(reason, ((LineSenderSchemaException) e.getCause()).getReason()); + } + } + + private static File taskTempRoot() { + File root = new File("target/schema-coordination-2.7/tmp").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class DescribeHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger requests = new AtomicInteger(); + private final CountDownLatch firstRequest = new CountDownLatch(1); + private volatile boolean reply; + private volatile boolean durableBeforeReply; + private volatile boolean malformedReply; + private TestWebSocketServer.ClientHandler client; + private long firstRequestId; + private final int result; + + private DescribeHandler(boolean reply) { + this(reply, QwpSchemaProtocol.RESULT_KNOWN); + } + + private DescribeHandler(boolean reply, int result) { + this.reply = reply; + this.result = result; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, + ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).getInt()); + Assert.assertEquals(QwpSchemaProtocol.FLAG_CONTROL, data[5]); + long requestId = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).getLong(QwpConstants.HEADER_SIZE + 1); + this.client = client; + int requestCount = requests.incrementAndGet(); + notifyAll(); + if (requestCount == 1) { + firstRequestId = requestId; + firstRequest.countDown(); + } + if (reply) { + if (malformedReply) { + ByteBuffer malformed = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN); + sendRaw(client, malformed.putInt(QwpConstants.MAGIC_MESSAGE).array()); + return; + } + if (durableBeforeReply) { + durableBeforeReply = false; + sendRaw(client, new byte[]{WebSocketResponse.STATUS_DURABLE_ACK, 0, 0}); + } + send(client, requestId, requests.get(), result); + } + } + + synchronized void replyToFirst() throws IOException { + send(client, firstRequestId, 1, result); + } + + boolean awaitFirstRequest() throws InterruptedException { + return firstRequest.await(5, TimeUnit.SECONDS); + } + + synchronized boolean awaitRequests(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (requests.get() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return true; + } + + synchronized void sendFeedback(String tableName, int result) { + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + byte[] schema = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) result).array(); + ByteBuffer b = ByteBuffer.allocate(11 + 2 + 2 + name.length + 4 + schema.length) + .order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(0).putShort((short) 0).putShort((short) 1) + .putShort((short) name.length).put(name).putInt(schema.length).put(schema); + sendRaw(client, b.array()); + } + + synchronized void sendInvalidation() { + ByteBuffer b = ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)) + .putLong(0).putShort((short) 0); + sendRaw(client, b.array()); + } + + private static void send(TestWebSocketServer.ClientHandler client, long requestId, int tableId, int result) { + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1; + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + payloadLength += 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + } + ByteBuffer b = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + b.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payloadLength).put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId).put((byte) result); + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + b.putInt(tableId).putLong(1).putShort((short) -1).putShort((short) 1) + .putShort((short) column.length).put(column).putInt(ColumnType.LONG).putShort((short) 0); + } + sendRaw(client, b.array()); + } + + private static void sendRaw(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(data); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java new file mode 100644 index 000000000..079941942 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -0,0 +1,454 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSchemaLifecycleTest { + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testCloseReleasesLongDeadlineWaiterBeforeLoopStarts() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("close-before-start"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + Lookup lookup = lookup(loop, "never-started", Long.MAX_VALUE, null); + Assert.assertTrue(lookup.entered.await(5, TimeUnit.SECONDS)); + loop.close(); + lookup.join(); + assertUnavailable(lookup.error.get()); + } + } + + @Test + public void testCloseReleasesLookupAlreadySentByActiveLoop() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("close-active"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup lookup = lookup(loop, "active", Long.MAX_VALUE, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + loop.close(); + lookup.join(); + assertUnavailable(lookup.error.get()); + } + } + + @Test + public void testInterruptionRestoresFlagAndReleasesSlotForNextLookup() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("interrupt"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup interrupted = lookup(loop, "first", Long.MAX_VALUE, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + interrupted.thread.interrupt(); + interrupted.join(); + assertUnavailable(interrupted.error.get()); + Assert.assertTrue(interrupted.interrupted.get()); + + handler.reply = true; + QwpSchemaResponse next = loop.resolveSchema("second", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, next.getResult()); + Assert.assertEquals("second", handler.lastName.get()); + } + } + + @Test + public void testDisconnectFailsCurrentLookupRetainsCacheAndQueuesMissForReplacement() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + QwpSchemaResponse cached = loop.resolveSchema("cached", 5_000); + handler.disconnectName = "lost"; + Lookup lost = lookup(loop, "lost", 5_000, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lost.join(); + assertUnavailable(lost.error.get()); + + Assert.assertSame(cached, loop.resolveSchema("CACHED", 5_000)); + CountDownLatch oneDone = new CountDownLatch(1); + Lookup first = lookup(loop, "fresh-one", 5_000, oneDone); + Lookup second = lookup(loop, "fresh-two", 5_000, oneDone); + Assert.assertTrue(oneDone.await(5, TimeUnit.SECONDS)); + reconnect.allow.countDown(); + first.join(); + second.join(); + Lookup successful = first.response.get() != null ? first : second; + Lookup busy = successful == first ? second : first; + Assert.assertNotNull(successful.response.get()); + assertUnavailable(busy.error.get()); + Assert.assertSame(successful.response.get(), loop.resolveSchema( + handler.lastName.get(), 5_000)); + + int requestsBeforeCacheProbe = handler.requests; + QwpSchemaResponse replacement = loop.resolveSchema("cached", 5_000); + Assert.assertTrue(replacement.getRequestId() > 0); + Assert.assertNotSame(cached, replacement); + Assert.assertEquals(requestsBeforeCacheProbe + 1, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testAsyncInitialConnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("async-initial")) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(null, engine, reconnect)) { + loop.start(); + Lookup lookup = lookup(loop, "deadline", 25, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + reconnect.allow.countDown(); + Assert.assertTrue(reconnect.connected.await(5, TimeUnit.SECONDS)); + QwpSchemaResponse probe = loop.resolveSchema("probe", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, probe.getResult()); + Assert.assertEquals(1, handler.requests); + Assert.assertEquals("probe", handler.lastName.get()); + } finally { + reconnect.allow.countDown(); + } + } + } + + @Test + public void testMalformedTimedOutReplyDoesNotPoisonIdleConnection() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("stale-idle"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup stale = lookup(loop, "stale", 1_000, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + stale.join(); + assertUnavailable(stale.error.get()); + handler.sendMalformedLastRequestAndFeedback("idle_barrier"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("idle_barrier", 0).getResult()); + break; + } catch (LineSenderSchemaException e) { + assertUnavailable(e); + if (System.nanoTime() >= deadline) { + Assert.fail("schema feedback barrier was not observed"); + } + Thread.yield(); + } + } + Assert.assertEquals(1, handler.requests); + handler.reply = true; + QwpSchemaResponse response = loop.resolveSchema("usable", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, response.getResult()); + Assert.assertEquals("usable", handler.lastName.get()); + Assert.assertEquals(1, server.handshakeCount()); + } + } + + @Test + public void testMalformedOldReplyDoesNotFailNewLookupOrPopulateOldCache() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("stale-pending"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup stale = lookup(loop, "stale", 1_000, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + stale.join(); + assertUnavailable(stale.error.get()); + handler.sendStaleBeforeNextReply = true; + QwpSchemaResponse current = loop.resolveSchema("current", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, current.getResult()); + Assert.assertEquals(2, current.getTableId()); + Assert.assertEquals(1, server.handshakeCount()); + + handler.reply = true; + QwpSchemaResponse staleProbe = loop.resolveSchema("stale", 5_000); + Assert.assertEquals(3, staleProbe.getTableId()); + Assert.assertEquals(3, handler.requests); + } + } + + @Test + public void testMalformedMatchingReplyFailsTransportAndReleasesWaiter() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("malformed-current"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup lookup = lookup(loop, "current", Long.MAX_VALUE, null); + lookup.join(); + assertUnavailable(lookup.error.get()); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + loop.checkError(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Invalid schema control response:")); + break; + } + if (System.nanoTime() >= deadline) { + Assert.fail("malformed schema response did not publish the transport error"); + } + Thread.yield(); + } + } + } + + private CursorSendEngine engine(String name) throws IOException { + return new CursorSendEngine(temp.newFolder(name).getAbsolutePath(), 1 << 20); + } + + private static CursorWebSocketSendLoop loop( + WebSocketClient client, + CursorSendEngine engine, + CursorWebSocketSendLoop.ReconnectFactory reconnect + ) { + return new CursorWebSocketSendLoop( + client, engine, 0, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, reconnect, 1, 4); + } + + private static Lookup lookup( + CursorWebSocketSendLoop loop, + String table, + long timeoutMillis, + CountDownLatch completed + ) { + Lookup lookup = new Lookup(); + lookup.thread = new Thread(() -> { + lookup.entered.countDown(); + try { + lookup.response.set(loop.resolveSchema(table, timeoutMillis)); + } catch (Throwable e) { + lookup.error.set(e); + } finally { + lookup.interrupted.set(Thread.currentThread().isInterrupted()); + if (completed != null) { + completed.countDown(); + } + } + }, "schema-lookup-" + table); + lookup.thread.start(); + return lookup; + } + + private static void assertUnavailable(Throwable error) { + Assert.assertTrue(String.valueOf(error), error instanceof LineSenderSchemaException); + Assert.assertEquals( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + ((LineSenderSchemaException) error).getReason()); + } + + private static TestWebSocketServer server(SchemaHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static WebSocketClient connect(int port) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + client.requestQwpSchema(); + client.upgrade("/write/v4", 5_000, null); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + private static File taskTempRoot() { + File root = new File("target/schema-coordination-2.7/tmp").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class Lookup { + private final AtomicReference error = new AtomicReference<>(); + private final CountDownLatch entered = new CountDownLatch(1); + private final AtomicBoolean interrupted = new AtomicBoolean(); + private final AtomicReference response = new AtomicReference<>(); + private Thread thread; + + private void join() throws InterruptedException { + thread.join(5_000); + Assert.assertFalse("lookup thread did not finish", thread.isAlive()); + } + } + + private static final class GatedReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final CountDownLatch allow = new CountDownLatch(1); + private final CountDownLatch attempted = new CountDownLatch(1); + private final CountDownLatch connected = new CountDownLatch(1); + private final int port; + + private GatedReconnectFactory(int port) { + this.port = port; + } + + @Override + public WebSocketClient reconnect() { + attempted.countDown(); + try { + if (!allow.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("replacement was not released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + WebSocketClient client = connect(port); + connected.countDown(); + return client; + } + + @Override + public void requireSchema() { + } + } + + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicReference lastName = new AtomicReference<>(); + private final CountDownLatch requestSeen = new CountDownLatch(1); + private volatile String disconnectName; + private volatile TestWebSocketServer.ClientHandler lastClient; + private volatile long lastRequestId; + private volatile boolean malformedReply; + private volatile boolean reply; + private volatile boolean sendStaleBeforeNextReply; + private volatile int requests; + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + ByteBuffer input = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = input.getLong(QwpConstants.HEADER_SIZE + 1); + int nameLength = input.getShort(QwpConstants.HEADER_SIZE + 9) & 0xffff; + String name = new String(data, QwpConstants.HEADER_SIZE + 11, nameLength, StandardCharsets.UTF_8); + requests++; + lastClient = client; + long staleRequestId = lastRequestId; + lastRequestId = requestId; + lastName.set(name); + requestSeen.countDown(); + if (name.equals(disconnectName)) { + try { + client.sendClose(1001, "test disconnect"); + } catch (IOException e) { + throw new AssertionError(e); + } + } else if (sendStaleBeforeNextReply) { + sendStaleBeforeNextReply = false; + sendMalformed(client, staleRequestId); + send(client, requestId, requests); + } else if (malformedReply) { + sendMalformed(client, requestId); + } else if (reply) { + send(client, requestId, requests); + } + } + + private synchronized void sendMalformedLastRequestAndFeedback(String tableName) { + sendMalformed(lastClient, lastRequestId); + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + byte[] schema = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0) + .put((byte) QwpSchemaProtocol.RESULT_MISSING).array(); + ByteBuffer feedback = ByteBuffer.allocate(11 + 2 + 2 + name.length + 4 + schema.length) + .order(ByteOrder.LITTLE_ENDIAN); + feedback.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(0).putShort((short) 0).putShort((short) 1) + .putShort((short) name.length).put(name).putInt(schema.length).put(schema); + try { + lastClient.sendBinary(feedback.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static void sendMalformed(TestWebSocketServer.ClientHandler client, long requestId) { + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + 9).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(9).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static void send(TestWebSocketServer.ClientHandler client, long requestId, int tableId) { + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payloadLength).put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId).put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(tableId) + .putLong(1).putShort((short) -1).putShort((short) 1) + .putShort((short) column.length).put(column).putInt(ColumnType.LONG).putShort((short) 0); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java deleted file mode 100644 index f2452dde0..000000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java +++ /dev/null @@ -1,251 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client.sf.cursor; - -import io.questdb.client.LineSenderServerException; -import io.questdb.client.SenderError; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.std.Unsafe; -import org.junit.Assert; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; - -/** - * Pinpointed tests for the latched-error contract on {@link CursorWebSocketSendLoop}: - * {@code recordFatal} → {@link CursorWebSocketSendLoop#getTerminalError} + - * {@link CursorWebSocketSendLoop#getLastTerminalServerError} + - * {@link CursorWebSocketSendLoop#checkError}. Bypasses the constructor entirely - * via {@code Unsafe.allocateInstance} to avoid the live wire/engine dependencies - * — the latch is a self-contained piece of state. - */ -public class CursorWebSocketSendLoopErrorLatchTest { - - @Test - public void testCheckErrorRethrowsLineSenderException() throws Exception { - // checkError must rethrow the SAME LineSenderException instance, not - // a wrapper. Producers depend on this so getServerError() works on - // typed throws. - CursorWebSocketSendLoop loop = newBareLoop(); - SenderError err = newSenderError(); - LineSenderServerException original = new LineSenderServerException(err); - setField(loop, "terminalError", original); - Assert.assertNull(loop.getSynchronouslySurfacedError()); - - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrown) { - Assert.assertSame("checkError must rethrow LineSenderException unchanged", - original, thrown); - Assert.assertSame(err, - ((LineSenderServerException) thrown).getServerError()); - } - Assert.assertSame("checkError must mark the exact latched throwable as user-owned", - original, loop.getSynchronouslySurfacedError()); - // a synchronously surfaced latch is owned -- the close() safety net - // must stay silent now - loop.checkUnsurfacedError(); - } - - @Test - public void testRecordFatalWrapsNonLineSenderThrowableOnce() throws Exception { - // Non-LineSenderException causes (NPE, IOException, etc.) are wrapped - // in a LineSenderException ONCE, at latch time -- so producers always - // see one exception type AND every rethrow delivers the same instance, - // which close() relies on to suppress double-signals by identity. - CursorWebSocketSendLoop loop = newBareLoop(); - Throwable raw = new RuntimeException("oh no"); - invokeRecordFatal(loop, raw); - Assert.assertNull(loop.getSynchronouslySurfacedError()); - - LineSenderException first = null; - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrown) { - Assert.assertNotSame(raw, thrown); - Assert.assertSame(raw, thrown.getCause()); - Assert.assertTrue(thrown.getMessage().contains("oh no")); - first = thrown; - } - Assert.assertSame("the latch must hold the wrapper, not the raw cause", - first, loop.getTerminalError()); - Assert.assertSame("ownership tracks the latched wrapper", - first, loop.getSynchronouslySurfacedError()); - loop.checkUnsurfacedError(); // owned -> silent - - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrownAgain) { - Assert.assertSame("every rethrow must deliver the same instance", - first, thrownAgain); - } - } - - @Test - public void testCheckErrorIsNoopWhenNoLatch() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Assert.assertNull(loop.getTerminalError()); - loop.checkError(); // must not throw - } - - @Test - public void testCheckUnsurfacedErrorThrowsOnceThenStaysSilent() throws Exception { - // The close() safety net: an unowned latch must rethrow exactly like - // checkError; once any synchronous caller has owned the error, it - // must stay silent -- unlike checkError, which keeps throwing. - CursorWebSocketSendLoop loop = newBareLoop(); - loop.checkUnsurfacedError(); // no latch -> silent - - LineSenderException e = new LineSenderException("wire fail"); - setField(loop, "terminalError", e); - try { - loop.checkUnsurfacedError(); - Assert.fail("an unowned latch must rethrow from the safety net"); - } catch (LineSenderException thrown) { - Assert.assertSame(e, thrown); - } - - loop.checkUnsurfacedError(); // the throw above made the caller owner -> silent - - try { - loop.checkError(); - Assert.fail("checkError must keep rethrowing a latched error"); - } catch (LineSenderException thrown) { - Assert.assertSame(e, thrown); - } - } - - @Test - public void testGetTerminalErrorReturnsLatchedThrowable() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Throwable e = new LineSenderException("boom"); - setField(loop, "terminalError", e); - Assert.assertSame(e, loop.getTerminalError()); - } - - @Test - public void testGetTerminalErrorIsNullBeforeAnyFailure() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Assert.assertNull("loops with no latched error must report null", - loop.getTerminalError()); - } - - @Test - public void testRecordFatalLatchesThrowableOnly() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - // running must be true initially so we can verify recordFatal flips it. - setField(loop, "running", true); - Throwable e = new LineSenderException("wire fail"); - - invokeRecordFatal(loop, e); - - Assert.assertSame(e, loop.getTerminalError()); - Assert.assertNull("typed payload must be null for a wire-level fatal", - loop.getLastTerminalServerError()); - Assert.assertFalse("recordFatal must stop the loop", - (Boolean) getField(loop, "running")); - } - - @Test - public void testRecordFatalLatchesBothThrowableAndSenderError() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - setField(loop, "running", true); - SenderError err = newSenderError(); - LineSenderServerException ex = new LineSenderServerException(err); - - invokeRecordFatal(loop, ex); - - Assert.assertSame(ex, loop.getTerminalError()); - Assert.assertSame("typed payload is derived from the latched LineSenderServerException", - err, loop.getLastTerminalServerError()); - Assert.assertFalse((Boolean) getField(loop, "running")); - } - - @Test - public void testRecordFatalIsIdempotent() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - setField(loop, "running", true); - SenderError firstErr = newSenderError(); - SenderError secondErr = newSenderError(); - LineSenderServerException first = new LineSenderServerException(firstErr); - LineSenderServerException second = new LineSenderServerException(secondErr); - - invokeRecordFatal(loop, first); - invokeRecordFatal(loop, second); - - // Only the first failure latches — subsequent calls must not - // overwrite, otherwise a follow-on cascade would mask the original - // root cause. - Assert.assertSame("first throwable must remain latched", - first, loop.getTerminalError()); - Assert.assertSame("first SenderError must remain latched", - firstErr, loop.getLastTerminalServerError()); - } - - private static SenderError newSenderError() { - return new SenderError( - SenderError.Category.SCHEMA_MISMATCH, - SenderError.Policy.TERMINAL, - 0x03, - "test-msg", - 7L, - 100L, 100L, - "tbl", - System.nanoTime() - ); - } - - private static CursorWebSocketSendLoop newBareLoop() throws Exception { - // Bypass the real constructor — we don't need a wire client or engine - // to test the latched-error contract. - return (CursorWebSocketSendLoop) Unsafe.getUnsafe() - .allocateInstance(CursorWebSocketSendLoop.class); - } - - private static void setField(Object target, String name, Object value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.set(target, value); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - private static void invokeRecordFatal(CursorWebSocketSendLoop loop, Throwable t) - throws Exception { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( - "recordFatal", Throwable.class); - m.setAccessible(true); - m.invoke(loop, t); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java index 130f60b44..4a8b156f1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java @@ -25,33 +25,68 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; /** * Regression coverage (M3): {@code catch (Throwable)} in the reconnect * machinery used to swallow {@link java.lang.Error} (OOM, LinkageError, * StackOverflowError) into an indefinite "transport outage" retry with only * a throttled, possibly-null-message WARN as a trace. A JVM/programming - * failure is not a transport outage -- retrying cannot clear it -- so every - * retry loop must rethrow {@code Error}, after latching it as terminal where - * a producer could otherwise hang in {@code checkError()}. - *

      - * Uses the same {@code Unsafe.allocateInstance} bare-loop pattern as - * {@link CursorWebSocketSendLoopErrorLatchTest}: the retry loops only touch - * the fields wired below, so no live wire client or engine is needed. + * failure is not a transport outage -- retrying cannot clear it -- so the + * blocking connection helper must rethrow {@code Error} immediately and the + * background loop must latch it for the producer before stopping. */ public class CursorWebSocketSendLoopJvmErrorTest { + @Test(timeout = 30_000) + public void testIoLoopLatchesReconnectJvmErrorAndShutsDown() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + LinkageError injected = new LinkageError("simulated JVM failure"); + try (CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024)) { + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + attempts.incrementAndGet(); + throw injected; + }, + 1L, + 4L, + false + ); + try { + loop.start(); + long deadlineNanos = System.nanoTime() + 5_000_000_000L; + while ((loop.getTerminalError() == null || loop.isRunning()) + && System.nanoTime() < deadlineNanos) { + Thread.yield(); + } + + Throwable terminal = loop.getTerminalError(); + Assert.assertNotNull("I/O loop did not latch the JVM Error", terminal); + Assert.assertEquals("a JVM Error must not be retried", 1, attempts.get()); + Assert.assertFalse("a terminal JVM Error must stop the loop", loop.isRunning()); + try { + loop.checkError(); + Assert.fail("producer-facing checkError() must surface the terminal failure"); + } catch (LineSenderException thrown) { + Assert.assertSame("checkError() must rethrow the latched wrapper", terminal, thrown); + Assert.assertSame("the wrapper must retain the original JVM Error", injected, thrown.getCause()); + } + } finally { + // Also proves that the real I/O-thread shutdown latch is released. + loop.close(); + } + } + } + @Test public void testConnectWithRetryPropagatesJvmError() { // The budgeted blocking initial-connect helper must not burn the @@ -75,110 +110,4 @@ public void testConnectWithRetryPropagatesJvmError() { Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); } - @Test - public void testConnectLoopPropagatesJvmErrorAndLatchesTerminal() throws Exception { - // The background per-outage reconnect loop must (1) latch the Error - // as terminal FIRST -- a producer parked in checkError() would - // otherwise never observe the failure -- and (2) rethrow so the I/O - // thread dies loudly instead of reconnect-looping forever. - CursorWebSocketSendLoop loop = newBareLoop(); - AtomicInteger attempts = new AtomicInteger(); - wireReconnectPlumbing(loop, attempts); - - Method connectLoop = CursorWebSocketSendLoop.class.getDeclaredMethod( - "connectLoop", Throwable.class, String.class, long.class); - connectLoop.setAccessible(true); - try { - connectLoop.invoke(loop, new LineSenderException("initial wire failure"), "reconnect", 0L); - Assert.fail("a JVM Error must escape connectLoop, not be retried"); - } catch (InvocationTargetException ite) { - Assert.assertTrue("expected LinkageError, got " + ite.getCause(), - ite.getCause() instanceof LinkageError); - } - Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); - assertErrorLatchedAndStopped(loop); - } - - @Test - public void testIoLoopDoesNotFunnelJvmErrorIntoReconnect() throws Exception { - // ioLoop's catch (Throwable) used to funnel EVERYTHING into - // fail(t) -> connectLoop(t, "reconnect"). An Error must instead be - // latched as terminal and rethrown; the finally still counts down - // the shutdown latch so close() cannot hang on the dead thread. - CursorWebSocketSendLoop loop = newBareLoop(); - AtomicInteger attempts = new AtomicInteger(); - wireReconnectPlumbing(loop, attempts); - CountDownLatch shutdownLatch = new CountDownLatch(1); - setField(loop, "shutdownLatch", shutdownLatch); - // client == null + running routes ioLoop into attemptInitialConnect - // -> connectLoop -> the throwing factory, exercising the full funnel. - - Method ioLoop = CursorWebSocketSendLoop.class.getDeclaredMethod("ioLoop"); - ioLoop.setAccessible(true); - try { - ioLoop.invoke(loop); - Assert.fail("a JVM Error must escape ioLoop, not re-enter the reconnect loop"); - } catch (InvocationTargetException ite) { - Assert.assertTrue("expected LinkageError, got " + ite.getCause(), - ite.getCause() instanceof LinkageError); - } - Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); - Assert.assertEquals("shutdown latch must count down so close() cannot hang", - 0L, shutdownLatch.getCount()); - assertErrorLatchedAndStopped(loop); - } - - private static void assertErrorLatchedAndStopped(CursorWebSocketSendLoop loop) - throws Exception { - Throwable terminal = loop.getTerminalError(); - Assert.assertNotNull("Error must be latched as terminal for checkError()", terminal); - Assert.assertTrue("latch wraps the raw Error once", - terminal instanceof LineSenderException); - Assert.assertTrue("latched cause must be the original Error", - terminal.getCause() instanceof LinkageError); - Assert.assertFalse("recordFatal must stop the loop", - (Boolean) getField(loop, "running")); - try { - loop.checkError(); - Assert.fail("producer-facing checkError must surface the latched terminal"); - } catch (LineSenderException thrown) { - Assert.assertSame(terminal, thrown); - } - } - - /** - * Wires the minimal state the reconnect paths dereference: a factory - * throwing {@link LinkageError} on every attempt, live {@code running}, - * and the attempt counters (field initializers do not run under - * {@code Unsafe.allocateInstance}). - */ - private static void wireReconnectPlumbing(CursorWebSocketSendLoop loop, - AtomicInteger attempts) throws Exception { - CursorWebSocketSendLoop.ReconnectFactory factory = () -> { - attempts.incrementAndGet(); - throw new LinkageError("simulated JVM failure"); - }; - setField(loop, "reconnectFactory", factory); - setField(loop, "running", true); - setField(loop, "totalReconnectAttempts", new AtomicLong()); - setField(loop, "totalReconnects", new AtomicLong()); - } - - private static CursorWebSocketSendLoop newBareLoop() throws Exception { - // Bypass the real constructor -- no wire client or engine needed. - return (CursorWebSocketSendLoop) Unsafe.getUnsafe() - .allocateInstance(CursorWebSocketSendLoop.class); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - private static void setField(Object target, String name, Object value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.set(target, value); - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java new file mode 100644 index 000000000..365ee32a1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -0,0 +1,562 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class QwpSchemaReplayNetworkTest { + private static final int SEGMENT_SIZE = 1 << 20; + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testBackgroundDrainerRejectsUnconfirmedClientReturnedByFactoryWithoutQuarantine() throws Exception { + File root = temp.newFolder("drainer-old-peer"); + String slot = new File(root, "ghost").getAbsolutePath(); + seedMixed(slot); + RecordingHandler oldHandler = new RecordingHandler(false, "replay"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + IgnoringFactory factory = new IgnoringFactory(oldPeer.getPort(), 2); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, SEGMENT_SIZE, 4L * SEGMENT_SIZE, factory, + 5_000, 1, 4, false, 0 + ); + AtomicInteger errors = new AtomicInteger(); + drainer.setErrorSink(error -> errors.incrementAndGet()); + Thread thread = new Thread(drainer::run, "schema-replay-drainer"); + thread.start(); + try { + Assert.assertTrue("factory must return at least two unconfirmed clients", + factory.awaitAttempts(5, TimeUnit.SECONDS)); + } finally { + drainer.requestStop(); + thread.join(5_000); + } + Assert.assertFalse("drainer did not stop", thread.isAlive()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.STOPPED, drainer.outcome()); + Assert.assertEquals(0, oldHandler.frames.size()); + Assert.assertEquals(0, errors.get()); + } + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test + public void testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames() throws Exception { + File root = temp.newFolder("mixed"); + String slot = new File(root, "default").getAbsolutePath(); + byte[][] expected = seedMixed(slot); + + RecordingHandler oldHandler = new RecordingHandler(false, "replay"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertThrows(QwpSchemaCapabilityMismatchException.class, + () -> Sender.fromConfig(config(oldPeer.getPort(), root, true))); + Assert.assertEquals(0, oldHandler.frames.size()); + } + assertRetained(slot, 1); + assertNoQuarantine(root); + + RecordingHandler supportingHandler = new RecordingHandler(true, "replay"); + try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { + supportingPeer.setAdvertiseSchema(true); + supportingPeer.start(); + Assert.assertTrue(supportingPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(supportingPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals(2, supportingHandler.frames.size()); + Assert.assertArrayEquals(expected[0], supportingHandler.frames.get(0)); + Assert.assertArrayEquals(expected[1], supportingHandler.frames.get(1)); + assertNoQuarantine(root); + } + + @Test + public void testDeferredOnlySchemaTailIsRetainedByOldPeerThenAbortedLocally() throws Exception { + File root = temp.newFolder("deferred"); + String slot = new File(root, "default").getAbsolutePath(); + seedDeferredSchemaTail(slot); + + RecordingHandler oldHandler = new RecordingHandler(false, "deferred"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertThrows(QwpSchemaCapabilityMismatchException.class, + () -> Sender.fromConfig(config(oldPeer.getPort(), root, false))); + Assert.assertEquals(0, oldHandler.frames.size()); + } + assertRetained(slot, 0); + assertNoQuarantine(root); + + RecordingHandler supportingHandler = new RecordingHandler(true, "deferred"); + try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { + supportingPeer.setAdvertiseSchema(true); + supportingPeer.start(); + Assert.assertTrue(supportingPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(supportingPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals("an uncommitted recovered tail must be aborted, not transmitted", 0, supportingHandler.frames.size()); + assertNoQuarantine(root); + } + + @Test + public void testLiveSchemaAppendIsRetainedWhenConnectedPeerIsLegacyOnly() throws Exception { + File root = temp.newFolder("live-append-old-peer"); + String slot = new File(root, "default").getAbsolutePath(); + RecordingHandler oldHandler = new RecordingHandler(false, "live_append"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + WebSocketClient initialClient = connectLegacyClient(oldPeer.getPort(), false); + SchemaAwareFactory factory = new SchemaAwareFactory(oldPeer.getPort(), 2); + CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, engine, 0, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, 1, 4, false); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer table = table("live_append", 7)) { + loop.start(); + int length = encoder.encodeSchema(table, -1, -1); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + Assert.assertTrue("schema-required reconnects must remain bounded and observable", + factory.awaitAttempts(5, TimeUnit.SECONDS)); + Assert.assertTrue(loop.isRunning()); + Assert.assertNull(loop.getTerminalError()); + Assert.assertEquals("no extended frame may reach a legacy peer", 0, oldHandler.frames.size()); + Assert.assertEquals(-1, engine.ackedFsn()); + Assert.assertEquals(0, engine.publishedFsn()); + } finally { + loop.close(); + engine.close(); + } + } + assertRetained(slot, 0); + assertNoQuarantine(root); + } + + @Test + public void testSchemaFeedbackAckAdvancesRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-ack"); + seedMixed(new File(root, "default").getAbsolutePath()); + FeedbackHandler handler = new FeedbackHandler(false); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(peer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals(2, handler.frames.get()); + assertNoQuarantine(root); + } + + @Test + public void testMalformedSchemaFeedbackDoesNotAdvanceRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-malformed"); + String slot = new File(root, "default").getAbsolutePath(); + seedMixed(slot); + FeedbackHandler handler = new FeedbackHandler(true); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + Sender sender = Sender.fromConfig(config(peer.getPort(), root, false)); + try { + Assert.assertTrue("malformed feedback must be parsed, rejected, and trigger replay", + handler.awaitDistinctClients(5, TimeUnit.SECONDS)); + } finally { + sender.close(); + } + } + Assert.assertTrue(handler.frames.get() > 0); + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test + public void testInvalidSchemaColumnsDoNotAdvanceRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-invalid-columns"); + String slot = new File(root, "default").getAbsolutePath(); + seedMixed(slot); + FeedbackHandler handler = new FeedbackHandler(false, true); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + Sender sender = Sender.fromConfig(config(peer.getPort(), root, false)); + try { + Assert.assertTrue("invalid schema feedback must be rejected and replayed", + handler.awaitDistinctClients(5, TimeUnit.SECONDS)); + } finally { + sender.close(); + } + } + Assert.assertTrue(handler.frames.get() > 0); + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test + public void testSchemaRequirementIsSharedAcrossForegroundAndBackgroundFactories() throws Exception { + RecordingHandler oldHandler = new RecordingHandler(false, "unused"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + assertSchemaRequirementShared(oldPeer.getPort(), true); + assertSchemaRequirementShared(oldPeer.getPort(), false); + Assert.assertEquals(0, oldHandler.frames.size()); + } + } + + private static void assertNoQuarantine(File root) { + File[] files = root.listFiles(); + Assert.assertNotNull(files); + for (File file : files) { + Assert.assertFalse(file.getName(), file.getName().contains(".unreplayable-")); + Assert.assertFalse(file.getName(), file.getName().equals(OrphanScanner.FAILED_SENTINEL_NAME)); + File failed = new File(file, OrphanScanner.FAILED_SENTINEL_NAME); + Assert.assertFalse(failed.getAbsolutePath(), failed.exists()); + } + } + + private static WebSocketClient connectLegacyClient(int port, boolean requestSchema) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + if (requestSchema) { + client.requestQwpSchema(); + } + client.upgrade("/write/v4", 5_000, null); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + private static void assertRetained(String slot, long publishedFsn) { + try (CursorSendEngine recovered = new CursorSendEngine(slot, SEGMENT_SIZE)) { + Assert.assertTrue(recovered.requiresSchema()); + Assert.assertEquals(-1, recovered.ackedFsn()); + Assert.assertEquals(publishedFsn, recovered.publishedFsn()); + } + } + + private static void assertSchemaRequirementShared(int port, boolean foregroundFirst) throws Exception { + try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig( + "ws::addr=localhost:" + port + ';')) { + CursorWebSocketSendLoop.ReconnectFactory foreground; + CursorWebSocketSendLoop.ReconnectFactory background; + if (foregroundFirst) { + foreground = sender.newReconnectFactory(); + background = sender.newBackgroundReconnectFactory(() -> false); + foreground.requireSchema(); + assertSchemaMismatch(background); + } else { + background = sender.newBackgroundReconnectFactory(() -> false); + foreground = sender.newReconnectFactory(); + background.requireSchema(); + assertSchemaMismatch(foreground); + } + } + } + + private static void assertSchemaMismatch(CursorWebSocketSendLoop.ReconnectFactory factory) throws Exception { + WebSocketClient client = null; + try { + client = factory.reconnect(); + Assert.fail("an old peer must be rejected after either factory requires schema"); + } catch (QwpSchemaCapabilityMismatchException expected) { + // Expected: the requirement belongs to the sender owner, not one factory instance. + } finally { + if (client != null) { + client.close(); + } + } + } + + private static String config(int port, File root, boolean durableAck) { + return "ws::addr=localhost:" + port + + ";sf_dir=" + root.getAbsolutePath() + + ";sender_id=default;sf_max_segment_bytes=1m;request_durable_ack=" + + (durableAck ? "on" : "off") + ";close_flush_timeout_millis=0;"; + } + + private static byte[] copy(QwpWebSocketEncoder encoder, int length) { + byte[] bytes = new byte[length]; + long address = encoder.getBuffer().getBufferPtr(); + for (int i = 0; i < length; i++) { + bytes[i] = io.questdb.client.std.Unsafe.getUnsafe().getByte(address + i); + } + return bytes; + } + + private static byte[][] seedMixed(String slot) { + try (CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer legacy = table("replay", 1); + QwpTableBuffer schema = table("replay", 2)) { + byte[] first = copy(encoder, encoder.encode(legacy)); + Assert.assertEquals(0, first[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), first.length)); + byte[] second = copy(encoder, encoder.encodeSchema(schema, -1, -1)); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, second[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), second.length)); + return new byte[][]{first, second}; + } + } + + private static void seedDeferredSchemaTail(String slot) { + try (CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer table = table("deferred", 1)) { + encoder.setDeferCommit(true); + int length = encoder.encodeSchema(table, -1, -1); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + } + } + + private static QwpTableBuffer table(String name, long value) { + QwpTableBuffer table = new QwpTableBuffer(name); + table.getOrCreateColumn("n", QwpConstants.TYPE_LONG, true).addLong(value); + table.nextRow(); + return table; + } + + private static File taskTempRoot() { + File root = new File("target/schema-replay-network").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class RecordingHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledge; + private final List frames = new ArrayList<>(); + private final String tableName; + + private RecordingHandler(boolean acknowledge, String tableName) { + this.acknowledge = acknowledge; + this.tableName = tableName; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + int sequence = frames.size(); + frames.add(data); + if (acknowledge) { + try { + client.sendBinary(ok(sequence)); + client.sendBinary(durable(sequence)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private byte[] durable(long seqTxn) { + byte[] name = tableName.getBytes(java.nio.charset.StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.allocate(1 + 2 + 2 + name.length + 8).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(WebSocketResponse.STATUS_DURABLE_ACK).putShort((short) 1).putShort((short) name.length).put(name).putLong(seqTxn); + return buffer.array(); + } + + private static byte[] ok(long sequence) { + return ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN) + .put(WebSocketResponse.STATUS_OK).putLong(sequence).putShort((short) 0).array(); + } + } + + private static final class IgnoringFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final CountDownLatch attemptsLatch; + private final int port; + + private IgnoringFactory(int port, int expectedAttempts) { + this.port = port; + this.attemptsLatch = new CountDownLatch(expectedAttempts); + } + + boolean awaitAttempts(long timeout, TimeUnit unit) throws InterruptedException { + return attemptsLatch.await(timeout, unit); + } + + @Override + public WebSocketClient reconnect() { + attempts.incrementAndGet(); + attemptsLatch.countDown(); + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + client.upgrade("/write/v4", 5_000, null); + Assert.assertFalse(client.isQwpSchemaEnabled()); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + @Override + public void requireSchema() { + // Deliberately ignored: returned-client validation must still reject it. + } + } + + private static final class FeedbackHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger frames = new AtomicInteger(); + private final CountDownLatch distinctClients; + private final boolean malformed; + private final boolean invalidColumns; + private TestWebSocketServer.ClientHandler lastClient; + + private FeedbackHandler(boolean malformed) { + this(malformed, false); + } + + private FeedbackHandler(boolean malformed, boolean invalidColumns) { + this.malformed = malformed; + this.invalidColumns = invalidColumns; + this.distinctClients = new CountDownLatch(malformed || invalidColumns ? 2 : 1); + } + + private boolean awaitDistinctClients(long timeout, TimeUnit unit) throws InterruptedException { + return distinctClients.await(timeout, unit); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + int sequence = frames.getAndIncrement(); + byte[] response = schemaFeedbackOk(sequence); + if (malformed) { + response = java.util.Arrays.copyOf(response, response.length - 1); + } else if (invalidColumns) { + response = schemaFeedbackInvalidColumns(sequence); + } + try { + client.sendBinary(response); + if (client != lastClient) { + lastClient = client; + distinctClients.countDown(); + } + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static byte[] schemaFeedbackOk(long sequence) { + byte[] tableName = "replay".getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.allocate(11 + 2 + 2 + tableName.length + 4 + 10) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES) + .putLong(sequence).putShort((short) 0).putShort((short) 1) + .putShort((short) tableName.length).put(tableName).putInt(10) + .put((byte) 2).putLong(0).put((byte) 1); + return buffer.array(); + } + + private static byte[] schemaFeedbackInvalidColumns(long sequence) { + byte[] tableName = "replay".getBytes(StandardCharsets.UTF_8); + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 26 + 2 * (2 + column.length + 6); + ByteBuffer buffer = ByteBuffer.allocate(11 + 2 + 2 + tableName.length + 4 + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES) + .putLong(sequence).putShort((short) 0).putShort((short) 1) + .putShort((short) tableName.length).put(tableName).putInt(payloadLength) + .put((byte) 2).putLong(0).put((byte) 0).putInt(1).putLong(1) + .putShort((short) -1).putShort((short) 2); + for (byte name : new byte[]{'x', 'X'}) { + buffer.putShort((short) 1).put(name).putInt(5).putShort((short) 0); + } + return buffer.array(); + } + } + + private static final class SchemaAwareFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final CountDownLatch attemptsLatch; + private final int port; + private volatile boolean schemaRequired; + + private SchemaAwareFactory(int port, int expectedAttempts) { + this.port = port; + this.attemptsLatch = new CountDownLatch(expectedAttempts); + } + + boolean awaitAttempts(long timeout, TimeUnit unit) throws InterruptedException { + return attemptsLatch.await(timeout, unit); + } + + @Override + public WebSocketClient reconnect() { + attemptsLatch.countDown(); + return connectLegacyClient(port, schemaRequired); + } + + @Override + public void requireSchema() { + schemaRequired = true; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java index d6ec8151b..5f09b51fa 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java @@ -25,8 +25,11 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import org.junit.Assert; @@ -35,6 +38,7 @@ import org.junit.rules.TemporaryFolder; import java.nio.file.Path; +import java.util.concurrent.TimeUnit; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -63,6 +67,105 @@ public void testUnterminatedSymbolLengthMarksRecoveredDeltaAsGap() throws Except assertMalformedDeltaMarksGap(new byte[]{0, 1, (byte) 0x80}); } + @Test + public void testSchemaRequirementRecoveredFromUnackedExtendedFrame() throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-schema-recovery").toPath(); + long payload = qwpFrame(QwpConstants.FLAG_SCHEMA); + try { + try (CursorSendEngine writer = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertFalse(writer.requiresSchema()); + Assert.assertEquals(0, writer.appendBlocking(payload, frameLength(payload))); + Assert.assertTrue(writer.requiresSchema()); + } + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertTrue(recovered.requiresSchema()); + } + } finally { + Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testAcknowledgedExtendedResidueDoesNotRequireSchema() throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-schema-acked-residue").toPath(); + long payload = qwpFrame(QwpConstants.FLAG_SCHEMA); + long legacy = qwpFrame((byte) 0); + try { + try (CursorSendEngine writer = new CursorSendEngine( + slot.toString(), 4_096, Long.MAX_VALUE, 30_000_000_000L, 1L)) { + Assert.assertEquals(0, writer.appendBlocking(payload, frameLength(payload))); + Assert.assertEquals(1, writer.appendBlocking(legacy, frameLength(legacy))); + Assert.assertTrue(writer.acknowledge(0)); + Assert.assertEquals(2, writer.appendBlocking(legacy, frameLength(legacy))); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try (AckWatermark watermark = AckWatermark.open(slot.toString())) { + if (watermark.read() >= 0) { + break; + } + } + Assert.assertTrue("ACK watermark was not checkpointed", System.nanoTime() < deadline); + Thread.yield(); + } + } + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertEquals(0, recovered.ackedFsn()); + Assert.assertEquals(2, recovered.publishedFsn()); + Assert.assertNotNull("acknowledged schema frame must remain represented", + recovered.findSegmentContaining(0)); + Assert.assertFalse(recovered.requiresSchema()); + } + } finally { + Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); + Unsafe.free(legacy, frameLength(legacy), MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testExtendedDeferredTailCannotRetireBeforeConfirmation() throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-schema-tail").toPath(); + long payload = qwpFrame((byte) (QwpConstants.FLAG_SCHEMA | QwpConstants.FLAG_DEFER_COMMIT)); + try { + try (CursorSendEngine writer = new CursorSendEngine(slot.toString(), 4_096)) { + writer.appendBlocking(payload, frameLength(payload)); + } + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertTrue(recovered.requiresSchema()); + Assert.assertFalse(recovered.retireRecoveredOrphanTailIfReady()); + Assert.assertEquals(-1, recovered.ackedFsn()); + Assert.assertTrue(recovered.retireRecoveredOrphanTailIfReady(true)); + Assert.assertEquals(0, recovered.ackedFsn()); + } + } finally { + Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static long qwpFrame(byte flags) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer table = new QwpTableBuffer("schema_table")) { + table.getOrCreateColumn("x", QwpConstants.TYPE_LONG, true).addLong(1); + table.nextRow(); + encoder.setDeferCommit((flags & QwpConstants.FLAG_DEFER_COMMIT) != 0); + int length = (flags & QwpConstants.FLAG_SCHEMA) != 0 + ? encoder.encodeSchema(table, 1, 1) + : encoder.encode(table); + long payload = Unsafe.malloc(length, MemoryTag.NATIVE_DEFAULT); + io.questdb.client.std.Vect.memcpy(payload, encoder.getBuffer().getBufferPtr(), length); + return payload; + } + } + + private static int frameLength(long address) { + return QwpConstants.HEADER_SIZE + Unsafe.getUnsafe().getInt(address + 8); + } + private void assertMalformedDeltaMarksGap(byte[] deltaSection) throws Exception { assertMemoryLeak(() -> { Path slot = temporaryFolder.newFolder("qwp-malformed-recovery").toPath(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java new file mode 100644 index 000000000..770188773 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java @@ -0,0 +1,535 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingBinaryTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/binary-input.tsv"; + + @Test + public void testConformanceCorpusUsesExactBinaryWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBinaryTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertEquals(line, "BINARY", fields[3]); + Assert.assertEquals(line, "VALID", fields[6]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + if ("BINARY".equals(fields[1])) { + binding.binaryColumn("value", bytes(fields[2])); + } else if ("STRING".equals(fields[1])) { + binding.stringColumn("value", "".equals(fields[2]) ? null : utf16(fields[2])); + } else { + throw new AssertionError(fields[1]); + } + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + if ("".equals(fields[4])) { + Assert.assertEquals(fields[0], 1, reader.byteValue()); + Assert.assertEquals(fields[0], 1, reader.byteValue()); + Assert.assertEquals(fields[0], 0, reader.intValue()); + } else { + byte[] expected = bytes(fields[4]); + if ("binary_all_256".equals(fields[0])) { + assertAllByteValues(bytes(fields[2])); + assertAllByteValues(expected); + assertAllByteValues(bytes(fields[5])); + } + Assert.assertEquals(fields[0], 0, reader.byteValue()); + Assert.assertEquals(fields[0], 0, reader.intValue()); + Assert.assertEquals(fields[0], expected.length, reader.intValue()); + Assert.assertArrayEquals(fields[0], expected, reader.bytes(expected.length)); + } + Assert.assertEquals(fields[0], encoder.getBuffer().getPosition(), reader.position()); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testAllBinaryOverloadsCopyBeforeReturning() throws Exception { + assertMemoryLeak(() -> { + long address = Unsafe.malloc(6, MemoryTag.NATIVE_DEFAULT); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + byte[] array = {1, 2}; + binding.binaryColumn("value", array); + array[0] = 9; + buffer.nextRow(); + + put(address, new byte[]{3, 4}); + DirectByteSlice slice = new DirectByteSlice().of(address, 2); + binding.binaryColumn("value", slice); + put(address, new byte[]{8, 8}); + slice.of(0, 0); + buffer.nextRow(); + + put(address, new byte[]{5, 6}); + binding.binaryColumn("value", address, 2); + put(address, new byte[]{7, 7}); + buffer.nextRow(); + Unsafe.free(address, 6, MemoryTag.NATIVE_DEFAULT); + address = 0; + binding.binaryColumn("value", 0, 0); + buffer.nextRow(); + + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 4, 1); + Assert.assertEquals(0, reader.byteValue()); + for (int offset : new int[]{0, 2, 4, 6, 6}) { + Assert.assertEquals(offset, reader.intValue()); + } + Assert.assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6}, reader.bytes(6)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } finally { + if (address != 0) { + Unsafe.free(address, 6, MemoryTag.NATIVE_DEFAULT); + } + } + }); + } + + @Test + public void testDuplicatePrecedesBinaryArgumentValidation() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.binaryColumn("value", new byte[]{42}) + .binaryColumn("value", (byte[]) null) + .binaryColumn("value", (DirectByteSlice) null) + .binaryColumn("value", 0, 1) + .binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1) + .stringColumn("value", new ThrowingCharSequence(new TestCharSequenceException())); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(42, reader.byteValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.stringColumn("value", "x").binaryColumn("value", (byte[]) null); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals('x', reader.byteValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testInvalidBinaryArgumentsAreTypedAndCheckedBeforeMemoryAccess() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + assertInvalid("binary value", () -> binding.binaryColumn("value", (byte[]) null)); + rollback(buffer); + assertInvalid("binary slice", () -> binding.binaryColumn("value", (DirectByteSlice) null)); + rollback(buffer); + assertInvalid("non-negative", () -> binding.binaryColumn("value", new DirectByteSlice().of(1, -1))); + rollback(buffer); + assertInvalid("pointer", () -> binding.binaryColumn("value", new DirectByteSlice().of(0, 1))); + rollback(buffer); + assertInvalid("non-negative", () -> binding.binaryColumn("value", 1, -1)); + rollback(buffer); + assertInvalid("pointer", () -> binding.binaryColumn("value", 0, 1)); + rollback(buffer); + assertInvalid("maximum", () -> binding.binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1)); + } + } + + @Test + public void testBitmapCrossesSixtyFourRowsWithPresentAndOmittedValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + for (int row = 0; row < 130; row++) { + if ((row & 1) == 0) { + binding.binaryColumn("value", new byte[]{(byte) row}); + } + buffer.nextRow(); + } + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 130, 1); + Assert.assertEquals(1, reader.byteValue()); + for (int i = 0; i < 16; i++) { + Assert.assertEquals(0xaa, reader.byteValue()); + } + Assert.assertEquals(0x02, reader.byteValue()); + for (int i = 0; i <= 65; i++) { + Assert.assertEquals(i, reader.intValue()); + } + for (int row = 0; row < 130; row += 2) { + Assert.assertEquals(row & 0xff, reader.byteValue()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.BINARY), column("existing_b", ColumnType.BINARY), + column("only_b", ColumnType.BINARY), column("c", ColumnType.BINARY)); + binding.binaryColumn("a", new byte[]{1}); + binding.binaryColumn("existing_b", new byte[]{2}); + buffer.nextRow(); + TestCharSequenceException existingFailure = new TestCharSequenceException(); + Assert.assertSame(existingFailure, Assert.assertThrows(TestCharSequenceException.class, + () -> binding.stringColumn("existing_b", new ThrowingCharSequence(existingFailure)))); + rollback(buffer); + TestCharSequenceException newColumnFailure = new TestCharSequenceException(); + Assert.assertSame(newColumnFailure, Assert.assertThrows(TestCharSequenceException.class, + () -> binding.stringColumn("only_b", new ThrowingCharSequence(newColumnFailure)))); + rollback(buffer); + binding.binaryColumn("c", new byte[]{3}); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 3); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.byteValue()); + Assert.assertEquals("existing_b", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.byteValue()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.byteValue()); + assertSparseBinary(reader, 2, 1); + assertSparseBinary(reader, 2, 2); + assertSparseBinary(reader, 1, 3); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testStringAppendIsExceptionAtomicForVarcharRuntimeAndError() throws Exception { + assertMemoryLeak(() -> { + for (Throwable failure : new Throwable[]{new TestCharSequenceException(), new TestCharSequenceError()}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.VARCHAR)); + binding.stringColumn("value", "a"); + buffer.nextRow(); + ThrowingCharSequence input = failure instanceof Error + ? new ThrowingCharSequence(failure, 4096, 300) + : new ThrowingCharSequence(failure); + Throwable actual = Assert.assertThrows(failure.getClass(), + () -> binding.stringColumn("value", input)); + Assert.assertSame(failure, actual); + rollback(buffer); + binding.stringColumn("value", "c"); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.byteValue()); + for (int offset : new int[]{0, 1, 2}) { + Assert.assertEquals(offset, reader.intValue()); + } + Assert.assertArrayEquals(new byte[]{'a', 'c'}, reader.bytes(2)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } + }); + } + + @Test + public void testResetClearAndSchemaGuards() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.binaryColumn("value", new byte[]{7}); + buffer.nextRow(); + buffer.reset(); + binding.binaryColumn("value", new byte[]{8}); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(8, reader.byteValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + Assert.assertThrows(IllegalStateException.class, () -> binding.binaryColumn("value", new byte[]{1})); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.STRING))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.INT))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.BINARY, new byte[]{1}))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, 0, column("value", ColumnType.BINARY))); + } + }); + } + + private static void assertSparseBinary(Reader reader, int bitmap, int value) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(bitmap, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(value, reader.byteValue()); + } + + private static void assertAllByteValues(byte[] bytes) { + Assert.assertEquals(256, bytes.length); + for (int i = 0; i < bytes.length; i++) { + Assert.assertEquals(i, bytes[i] & 0xff); + } + } + + private static void assertUnsupported(QwpSchemaBinding binding) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> binding.binaryColumn("value", new byte[]{1})); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, error.getReason()); + } + + private static void assertInvalid(String detail, Runnable runnable) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(error.getMessage(), LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("table=t")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=BINARY")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=BINARY")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains(detail)); + } + + private static void rollback(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static void put(long address, byte[] bytes) { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(address + i, bytes[i]); + } + } + + private static byte[] bytes(String hex) { + Assert.assertEquals(hex, 0, hex.length() & 1); + byte[] result = new byte[hex.length() / 2]; + for (int i = 0; i < result.length; i++) { + result[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16); + } + return result; + } + + private static String utf16(String hex) { + Assert.assertEquals(hex, 0, hex.length() & 3); + char[] chars = new char[hex.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + return tableReader(encoder, size, rows, columns, QwpConstants.TYPE_BINARY); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns, int wireType) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.byteValue()); + return reader; + } + + private static void skipTablePrefix(Reader reader, int rows, int columns) { + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN).putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(length).put(payload.array()); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + put(address, frame.array()); + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int byteValue() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 255; + } + + private byte[] bytes(int count) { + Assert.assertTrue(position + count <= limit); + byte[] bytes = new byte[count]; + for (int i = 0; i < count; i++) { + bytes[i] = (byte) byteValue(); + } + return bytes; + } + + private int intValue() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long longValue() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int count) { + Assert.assertTrue(position + count <= limit); + position += count; + } + + private String string() { + return new String(bytes(varint()), StandardCharsets.UTF_8); + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = byteValue(); + result |= (value & 127) << shift; + shift += 7; + } while ((value & 128) != 0); + return result; + } + } + + private static final class ThrowingCharSequence implements CharSequence { + private final Throwable failure; + private final int failureIndex; + private final int length; + + private ThrowingCharSequence(Throwable failure) { + this(failure, 2, 1); + } + + private ThrowingCharSequence(Throwable failure, int length, int failureIndex) { + this.failure = failure; + this.length = length; + this.failureIndex = failureIndex; + } + + @Override + public int length() { + return length; + } + + @Override + public char charAt(int index) { + if (index < failureIndex) { + return 'x'; + } + if (failure instanceof Error) { + throw (Error) failure; + } + throw (RuntimeException) failure; + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + } + + private static final class TestCharSequenceException extends RuntimeException { + } + + private static final class TestCharSequenceError extends Error { + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java new file mode 100644 index 000000000..1bf7185ed --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java @@ -0,0 +1,372 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingBooleanTest { + private static final String BOOLEAN_CORPUS = "/io/questdb/client/cutlass/qwp/boolean-to-target.tsv"; + private static final String STRING_CORPUS = "/io/questdb/client/cutlass/qwp/string-to-boolean.tsv"; + + @Test + public void testBooleanCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBooleanTest.class.getResourceAsStream(BOOLEAN_CORPUS); + Assert.assertNotNull(BOOLEAN_CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 6, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(fields[2]))); + Assert.assertTrue(fields[1], "TRUE".equals(fields[1]) || "FALSE".equals(fields[1])); + binding.boolColumn("value", "TRUE".equals(fields[1])); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(fields[3]), 1); + Assert.assertEquals(0, reader.byteValue()); + assertValue(reader, fields[3], fields[4]); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testStringCorpusParsesStrictAsciiAndUsesBooleanWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBooleanTest.class.getResourceAsStream(STRING_CORPUS); + Assert.assertNotNull(STRING_CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 4, fields.length); + CharSequence value = "".equals(fields[1]) ? null : utf16(fields[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BOOLEAN)); + if ("".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("value", value)); + } else { + binding.stringColumn("value", value); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 1); + if (value == null) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + } else { + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(Integer.parseInt(fields[2]), reader.byteValue()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(80, count); + }); + } + + @Test + public void testBooleanPackingPastEightAndSixtyFourWithOmissions() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BOOLEAN)); + for (int row = 0; row < 137; row++) { + if ((row & 1) == 0) { + binding.boolColumn("value", (row & 3) == 0); + } + buffer.nextRow(); + } + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 137); + Assert.assertEquals(1, reader.byteValue()); + byte[] nulls = new byte[18]; + for (int i = 0; i < 17; i++) nulls[i] = (byte) 0xaa; + Assert.assertArrayEquals(nulls, reader.bytes(nulls.length)); + byte[] expected = new byte[9]; + for (int i = 0; i < 8; i++) expected[i] = 0x55; + expected[8] = 0x15; + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + + buffer.reset(); + for (int row = 0; row < 70; row++) buffer.nextRow(); + Reader allNull = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 70); + Assert.assertEquals(1, allNull.byteValue()); + byte[] allNulls = new byte[9]; + for (int i = 0; i < 8; i++) allNulls[i] = (byte) 0xff; + allNulls[8] = 0x3f; + Assert.assertArrayEquals(allNulls, allNull.bytes(allNulls.length)); + Assert.assertEquals(encoder.getBuffer().getPosition(), allNull.position()); + } + }); + } + + @Test + public void testDuplicateRollbackFailedRowAndLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.BOOLEAN), column("bad", ColumnType.BOOLEAN), column("c", ColumnType.INT)); + binding.boolColumn("a", true); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "invalid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.boolColumn("c", true); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reader.byteValue()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + StringBuilder mutable = new StringBuilder("TrUe"); + binding.stringColumn("a", mutable).boolColumn("a", false); + mutable.replace(0, mutable.length(), "false"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reset, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reset.byteValue()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reset.byteValue()); + Assert.assertEquals(0, reset.byteValue()); + Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(resetSize, reset.position()); + buffer.clear(); + assertIllegalState(() -> binding.boolColumn("a", true)); + assertIllegalState(() -> binding.stringColumn("a", "true")); + } + }); + } + + @Test + public void testDuplicatePrecedesUnsupportedCrossSetterAndInvalidParse() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("flag", ColumnType.BOOLEAN), column("sym", ColumnType.SYMBOL)); + binding.boolColumn("flag", true).stringColumn("flag", "invalid"); + binding.stringColumn("sym", "first").boolColumn("sym", false); + buffer.nextRow(); + Assert.assertTrue(encoder.encodeSchema(buffer) > 0); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("sym", ColumnType.SYMBOL), column("future", ColumnType.BOOLEAN, new byte[]{1}), + column("flagged", ColumnType.BOOLEAN | 0x10000), column("plain", ColumnType.BOOLEAN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("sym", true)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.symbol("plain", null)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.longColumn("plain", Long.MIN_VALUE)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("future", true)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("flagged", "true")); + rollback(buffer); + binding.boolColumn("missing", true); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.BOOLEAN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("d", true)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("d", null)); + } + } + + private static void assertValue(Reader reader, String type, String expected) { + switch (type) { + case "BOOLEAN": reader.assertByte(Integer.parseInt(expected)); break; + case "BYTE": reader.assertByte(Integer.parseInt(expected)); break; + case "SHORT": Assert.assertEquals(Integer.parseInt(expected), reader.shortValue()); break; + case "INT": Assert.assertEquals(Integer.parseInt(expected), reader.intValue()); break; + case "LONG": Assert.assertEquals(Long.parseLong(expected), reader.longValue()); break; + case "FLOAT": Assert.assertEquals(Float.floatToRawIntBits(Float.parseFloat(expected)), Float.floatToRawIntBits(reader.floatValue())); break; + case "DOUBLE": Assert.assertEquals(Double.doubleToRawLongBits(Double.parseDouble(expected)), Double.doubleToRawLongBits(reader.doubleValue())); break; + case "VARCHAR": + byte[] bytes = hex(expected); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(bytes.length, reader.intValue()); + Assert.assertArrayEquals(bytes, reader.bytes(bytes.length)); + break; + default: throw new AssertionError("unknown wire type: " + type); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { return column(name, type, new byte[0]); } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) length += column.length; + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) payload.put(column); + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(length).put(payload.array()); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.byteValue()); + return reader; + } + + private static void skipTablePrefix(Reader reader, int rows, int columns) { + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + } + + private static int targetType(String value) { + switch (value) { + case "BOOLEAN": return ColumnType.BOOLEAN; + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + case "STRING": return ColumnType.STRING; + case "VARCHAR": return ColumnType.VARCHAR; + default: throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BOOLEAN": return QwpConstants.TYPE_BOOLEAN; + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + case "VARCHAR": return QwpConstants.TYPE_VARCHAR; + default: throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } + + private static String utf16(String value) { + Assert.assertEquals(0, value.length() & 3); + char[] chars = new char[value.length() / 4]; + for (int i = 0; i < chars.length; i++) chars[i] = (char) Integer.parseInt(value.substring(i * 4, i * 4 + 4), 16); + return new String(chars); + } + + private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } + private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; private final int limit; private int position; + private Reader(long address, int limit) { this.address = address; this.limit = limit; } + private void assertByte(int value) { Assert.assertEquals(value, byteValue()); } + private int byteValue() { Assert.assertTrue(position < limit); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } + private byte[] bytes(int length) { Assert.assertTrue(position + length <= limit); byte[] value = new byte[length]; for (int i = 0; i < length; i++) value[i] = (byte) byteValue(); return value; } + private short shortValue() { Assert.assertTrue(position + 2 <= limit); short v = Unsafe.getUnsafe().getShort(address + position); position += 2; return v; } + private int intValue() { Assert.assertTrue(position + 4 <= limit); int v = Unsafe.getUnsafe().getInt(address + position); position += 4; return v; } + private long longValue() { Assert.assertTrue(position + 8 <= limit); long v = Unsafe.getUnsafe().getLong(address + position); position += 8; return v; } + private float floatValue() { return Float.intBitsToFloat(intValue()); } + private double doubleValue() { return Double.longBitsToDouble(longValue()); } + private int position() { return position; } + private void skip(int length) { Assert.assertTrue(position + length <= limit); position += length; } + private String string() { return new String(bytes(varint()), StandardCharsets.UTF_8); } + private int varint() { int r=0,s=0,v; do { v=byteValue(); r|=(v&0x7f)<".equals(fields[2]) ? null : fromUtf16Hex(fields[2])); + } else { + throw new AssertionError("unknown input kind: " + fields[1]); + } + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, QwpConstants.TYPE_CHAR); + if ("".equals(fields[2])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(fields[3]), reader.u16()); + } + Assert.assertEquals(size, reader.position); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(23, count); + } + + @Test + public void testDuplicateOmissionRollbackResetAndInference() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, + column("a", ColumnType.CHAR), column("bad", ColumnType.UUID), column("c", ColumnType.CHAR))); + binding.stringColumn("a", "A").binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "B"); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("bad", 'C')); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "Z"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 3, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(6, reader.u8()); + Assert.assertEquals('A', reader.u16()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals('Z', reader.u16()); + Assert.assertEquals(size, reader.position); + + buffer.reset(); + binding.stringColumn("a", "Q"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals('Q', reset.u16()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "A"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.charColumn("value", 'A'); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_CHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + } + + @Test + public void testUnsupportedAndDesignatedTargetsRemainRejected() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, + column("value", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("value", 'A')); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("ts", 'A')); + } + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static String fromUtf16Hex(String hex) { + StringBuilder value = new StringBuilder(hex.length() / 4); + for (int i = 0; i < hex.length(); i += 4) { + value.append((char) Integer.parseInt(hex.substring(i, i + 4), 16)); + } + return value.toString(); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + Reader reader = tableHeader(encoder, size, 1, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int u16() { + return u8() | (u8() << 8); + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java new file mode 100644 index 000000000..910129744 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java @@ -0,0 +1,408 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv"; + private static final String HEADER = "# case_id\tsource_type\tinput_bits_hex\ttarget_type\t" + + "target_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\t" + + "expected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingFloatingDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[][] sourceTargets = new boolean[2][6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 12, fields.length); + Vector vector = new Vector(fields); + sourceTargets[vector.sourceIndex()][targetIndex(vector.targetType)] = true; + assertVector(vector); + count++; + } + } + Assert.assertEquals(60, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target pair " + source + '/' + target, + sourceTargets[source][target]); + } + } + }); + } + + @Test + public void testDuplicateNullRollbackAndOmissionLifecycle() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(3, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + // Duplicate suppression precedes both an invalid conversion and null normalization. + binding.floatColumn("value", 1.0f) + .doubleColumn("value", 1.25) + .doubleColumn("value", Double.NEGATIVE_INFINITY); + buffer.nextRow(); + + // A null first value also wins over a later invalid duplicate. + binding.doubleColumn("value", Double.longBitsToDouble(0x7ff8000000000042L)) + .floatColumn("value", 1.25f); + buffer.nextRow(); + + // Row B must disappear without disturbing completed A or following C. + binding.doubleColumn("only_b", 2.0); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.doubleColumn("value", 1.25)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.floatColumn("value", -2.5f); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, QwpConstants.TYPE_DECIMAL64, 4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0a, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-25, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL64, 1); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void assertInvalid(Vector vector) throws Exception { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> vector.append(binding, "value")); + Assert.assertEquals(vector.caseId, LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(vector.caseId, error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=value")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=" + vector.sourceType)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=DECIMAL")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + } + + private static void assertVector(Vector vector) throws Exception { + try { + if (vector.invalid()) { + assertInvalid(vector); + return; + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + vector.append(binding, "value"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); + Reader reader = tableReader(encoder, size, wireType(vector.targetType), 1); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static QwpSchemaResponse known(byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) -1).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + return reader; + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + return 5; + } + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final long inputBits; + private final String outcome; + private final String sourceType; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + sourceType = fields[1]; + Assert.assertTrue(sourceType.equals("FLOAT") || sourceType.equals("DOUBLE")); + inputBits = Long.parseUnsignedLong(fields[2], 16); + Assert.assertEquals(sourceType.equals("FLOAT") ? 8 : 16, fields[2].length()); + targetType = fields[3]; + targetPrecision = Integer.parseInt(fields[4]); + targetScale = Integer.parseInt(fields[5]); + outcome = fields[6]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) { + expectedLimbs[i] = parseHex(fields[7 + i]); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[7 + i]); + } + if (outcome.equals("NULL")) { + Assert.assertEquals("NULL", fields[11]); + } else if (outcome.equals("INVALID")) { + Assert.assertEquals("-", fields[11]); + } else { + Assert.assertFalse(fields[11].isEmpty()); + Assert.assertNotEquals("-", fields[11]); + } + } + + private void append(QwpSchemaBinding binding, String column) { + if ("FLOAT".equals(sourceType)) { + binding.floatColumn(column, Float.intBitsToFloat((int) inputBits)); + } else { + binding.doubleColumn(column, Double.longBitsToDouble(inputBits)); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private int sourceIndex() { + return "FLOAT".equals(sourceType) ? 0 : 1; + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java new file mode 100644 index 000000000..aa47bad4b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java @@ -0,0 +1,317 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingFloatingTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] f = line.split("\\t", -1); + Assert.assertEquals(line, 7, f.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(f[3]))); + Runnable append = () -> append(binding, f[1], f[2]); + if ("".equals(f[5])) { + Assert.assertEquals(f[0], "", f[6]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + } else { + append.run(); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[4]), 1, 1); + if ("".equals(f[5])) { + Assert.assertEquals(f[0], + "BYTE".equals(f[3]) || "SHORT".equals(f[3]) ? "0" : "NULL", f[6]); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + } else { + Assert.assertEquals(0, reader.byteValue()); + Assert.assertArrayEquals(f[0], hex(f[5]), reader.bytes(hex(f[5]).length)); + } + Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(350, count); + }); + } + + @Test + public void testNaNIsMissingWithOtherNullAndPresentValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DOUBLE)); + binding.floatColumn("value", Float.intBitsToFloat(0x7fc12345)); + buffer.nextRow(); + buffer.nextRow(); + binding.doubleColumn("value", -0.0d); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_DOUBLE, 3, 1); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(3, reader.byteValue()); + Assert.assertEquals(0x8000000000000000L, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testDuplicatePrecedesInvalidAndUnsupportedConversion() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("n", ColumnType.LONG), column("uuid", ColumnType.UUID)); + binding.floatColumn("n", 1).doubleColumn("n", Double.POSITIVE_INFINITY); + binding.uuidColumn("uuid", 1, 2).floatColumn("uuid", Float.NaN); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 1, 2); + Assert.assertEquals("n", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.byteValue()); + Assert.assertEquals("uuid", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.FLOAT), column("only_b", ColumnType.DOUBLE), + column("bad", ColumnType.BYTE), column("c", ColumnType.LONG)); + binding.doubleColumn("a", 1.5); + buffer.nextRow(); + binding.floatColumn("only_b", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.doubleColumn("bad", 128)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.floatColumn("c", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_FLOAT, reader.byteValue()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(Float.floatToRawIntBits(1.5f), reader.intValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(3, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testResetClearAndStaleBinding() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.FLOAT)); + binding.doubleColumn("value", 9.25); + buffer.nextRow(); + buffer.reset(); + binding.floatColumn("value", -0.0f); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_FLOAT, 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0x80000000, reader.intValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + assertIllegalState(() -> binding.floatColumn("value", 1)); + assertIllegalState(() -> binding.doubleColumn("value", 1)); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("uuid", ColumnType.UUID), column("future", ColumnType.FLOAT, new byte[]{1}), + column("flagged", ColumnType.DOUBLE | 0x10000), column("bool", ColumnType.BOOLEAN), + column("timestamp", ColumnType.TIMESTAMP), column("binary", ColumnType.BINARY), + column("byte", ColumnType.BYTE), column("short", ColumnType.SHORT)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("uuid", Float.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("future", Double.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("flagged", 1)); + rollback(buffer); + binding.doubleColumn("missing", 1); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("bool", Float.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("timestamp", Double.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("binary", Float.NaN)); + rollback(buffer); + assertInvalidContext(binding, buffer, "byte", true); + assertInvalidContext(binding, buffer, "short", false); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.DOUBLE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("d", Float.NaN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("d", Double.NaN)); + } + } + + private static void append(QwpSchemaBinding binding, String inputType, String bits) { + if ("FLOAT".equals(inputType)) { + binding.floatColumn("value", Float.intBitsToFloat((int) Long.parseLong(bits, 16))); + } else if ("DOUBLE".equals(inputType)) { + binding.doubleColumn("value", Double.longBitsToDouble(Long.parseUnsignedLong(bits, 16))); + } else { + throw new AssertionError("unknown input type: " + inputType); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { return column(name, type, new byte[0]); } + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) length += column.length; + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) payload.put(column); + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(length).put(payload.array()); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.byteValue()); + return reader; + } + + private static void skipTablePrefix(Reader reader, int rows, int columns) { + Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); Assert.assertEquals(columns, reader.varint()); + } + + private static int targetType(String value) { + switch (value) { + case "BYTE": return ColumnType.BYTE; case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BYTE": return QwpConstants.TYPE_BYTE; case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + Assert.assertEquals(value, 0, value.length() & 1); byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring((bytes.length - 1 - i) * 2, (bytes.length - i) * 2), 16); + return bytes; + } + + private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } + private static void assertInvalidContext(QwpSchemaBinding binding, QwpTableBuffer buffer, String column, boolean floatInput) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> { if (floatInput) binding.floatColumn(column, 128); else binding.doubleColumn(column, Double.POSITIVE_INFINITY); }); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=" + column)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=" + (floatInput ? "FLOAT" : "DOUBLE"))); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + column.toUpperCase())); + rollback(buffer); + } + private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; private final int limit; private int position; + private Reader(long address, int limit) { this.address=address; this.limit=limit; } + private int byteValue() { Assert.assertTrue(position { + InputStream stream = + QwpSchemaBindingFloatingTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = + new BufferedReader( + new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + boolean symbol = "SYMBOL".equals(fields[3]); + QwpSchemaBinding binding = + binding(buffer, column("value", targetType(fields[3]))); + append(binding, fields[1], fields[2], "value"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = + tableReader( + encoder, + size, + symbol + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR, + 1); + boolean missing = "".equals(fields[5]); + Assert.assertEquals(missing, "".equals(fields[6])); + Assert.assertEquals(missing ? 1 : 0, reader.u8()); + if (missing) { + Assert.assertEquals(1, reader.u8()); + if (symbol) { + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + } + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals( + fields[6], + new String(expected, StandardCharsets.UTF_8)); + if (symbol) { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals( + expected, reader.bytes(expected.length)); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError( + "case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(48, count); + }); + } + + @Test + public void testNaNDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak( + () -> { + for (int target : + new int[] {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = + target == ColumnType.SYMBOL + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR; + QwpSchemaBinding binding = binding(buffer, column("value", target)); + binding.floatColumn("value", Float.intBitsToFloat(0x7fc00002)) + .binaryColumn("value", new byte[] {1}); + buffer.nextRow(); + binding.doubleColumn("value", -0.0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("-0.0", reader.string()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(4, reader.i32()); + Assert.assertEquals("-0.0", reader.ascii(4)); + } + Assert.assertEquals(size, reader.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = + binding( + buffer, + column("a", ColumnType.STRING), + column("bad", ColumnType.UUID), + column("c", ColumnType.VARCHAR)); + binding.doubleColumn("a", 1e23); + buffer.nextRow(); + binding.floatColumn("a", 0.1f); + assertReason( + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleColumn("bad", 2.0)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.floatColumn("c", Float.intBitsToFloat(1)); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarchar(reader, 6, "1.0E23"); + assertVarchar(reader, 5, "1.401298464324817E-45"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.doubleColumn("a", Double.NEGATIVE_INFINITY); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + assertVarchar(reset, 0, "-Infinity"); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testSymbolFormatterScratchIsCopied() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.doubleColumn("value", 42.0); + buffer.nextRow(); + binding.doubleColumn("value", 43.0); + buffer.nextRow(); + binding.doubleColumn("value", 42.0); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("42.0", reader.string()); + Assert.assertEquals("43.0", reader.string()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testNativeInferenceAndUnsupportedTargets() { + assertNative("FLOAT", "value", 0x80000000L); + assertNative("DOUBLE", "value", 0x8000000000000000L); + for (int target : new int[] {ColumnType.TIMESTAMP, ColumnType.UUID, ColumnType.BOOLEAN}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", target)); + assertReason( + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> + binding.doubleColumn( + "value", Double.longBitsToDouble(0x7ff8000000000002L))); + } + } + } + + private static void assertNative(String inputType, String name, long bits) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + append( + binding, + inputType, + inputType.equals("FLOAT") + ? String.format("%08x", bits) + : String.format("%016x", bits), + name); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals( + inputType.equals("FLOAT") ? QwpConstants.TYPE_FLOAT : QwpConstants.TYPE_DOUBLE, + reader.u8()); + Assert.assertEquals(0, reader.u8()); + if (inputType.equals("FLOAT")) { + Assert.assertEquals((int) bits, reader.i32()); + } else { + Assert.assertEquals(bits, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + + private static void append(QwpSchemaBinding binding, String type, String bits, String name) { + if ("FLOAT".equals(type)) + binding.floatColumn(name, Float.intBitsToFloat((int) Long.parseUnsignedLong(bits, 16))); + else binding.doubleColumn(name, Double.longBitsToDouble(Long.parseUnsignedLong(bits, 16))); + } + + private static void assertVarchar(Reader reader, int bitmap, String expected) { + if (bitmap == 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + } + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length(), reader.i32()); + Assert.assertEquals(expected, reader.ascii(expected.length())); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6) + .order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length) + .put(bytes) + .putInt(type) + .putShort((short) 0) + .array(); + } + + private static QwpSchemaResponse missing() { + return decode( + ByteBuffer.allocate(10) + .order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(1) + .put((byte) QwpSchemaProtocol.RESULT_MISSING) + .array()); + } + + private static QwpSchemaResponse known(byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = + ByteBuffer.allocate(length) + .order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(1) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1) + .putLong(1) + .putShort((short) -1) + .putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = + ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) 1) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payload.length) + .put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) { + return ColumnType.STRING; + } + if ("VARCHAR".equals(value)) { + return ColumnType.VARCHAR; + } + if ("SYMBOL".equals(value)) { + return ColumnType.SYMBOL; + } + throw new AssertionError(value); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals( + reason, + Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private String ascii(int length) { + return new String(bytes(length), StandardCharsets.UTF_8); + } + + private byte[] bytes(int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = (byte) u8(); + } + return value; + } + + private int i32() { + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + return ascii(varint()); + } + + private byte[] stringBytes() { + return bytes(varint()); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0, shift = 0, value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java new file mode 100644 index 000000000..37af086da --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java @@ -0,0 +1,362 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-decimal.tsv"; + private static final String HEADER = "# case_id\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[] targets = new boolean[6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 11, fields.length); + Vector vector = new Vector(fields); + targets[targetIndex(vector.targetType)] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + if (vector.invalid()) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("value", vector.input)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } else { + binding.longColumn("value", vector.input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); + Reader reader = tableReader(encoder, size, wireType(vector.targetType)); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(39, count); + for (int i = 0; i < targets.length; i++) { + Assert.assertTrue("missing target " + i, targets[i]); + } + }); + } + + @Test + public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(2, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + binding.longColumn("value", 1).longColumn("value", 10); + buffer.nextRow(); + + binding.longColumn("only_b", 1); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("value", 10)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.longColumn("value", -1); + buffer.nextRow(); + binding.longColumn("value", Long.MIN_VALUE).longColumn("value", 1); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 4, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0c, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-10, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 1); + decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static void decimalDefinition(Reader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static QwpSchemaResponse known(byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) -1).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + return 5; + } + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + Reader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final long input; + private final String outcome; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + input = Long.parseLong(fields[1]); + targetType = fields[2]; + targetPrecision = Integer.parseInt(fields[3]); + targetScale = Integer.parseInt(fields[4]); + outcome = fields[5]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) { + expectedLimbs[i] = parseHex(fields[6 + i]); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[6 + i]); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java new file mode 100644 index 000000000..d6fbaa08d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java @@ -0,0 +1,400 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-text.tsv"; + + @Test + public void testCorpusUsesExactTextWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = wireType(fields[3]); + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); + binding.longColumn("value", Long.parseLong(fields[1])); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, wireType, 1); + if ("".equals(fields[4])) { + Assert.assertEquals("", fields[5]); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + if (wireType == QwpConstants.TYPE_VARCHAR) Assert.assertEquals(0, reader.intValue()); + else Assert.assertEquals(0, reader.varint()); + } else { + byte[] expected = hex(fields[4]); + Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals(0, reader.byteValue()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + } else { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = targetType == ColumnType.SYMBOL + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR; + QwpSchemaBinding binding = binding(buffer, column("value", targetType)); + binding.longColumn("value", Long.MIN_VALUE).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(5, reader.byteValue()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(2, reader.intValue()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.bytes(2)); + } else { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } + Assert.assertEquals(size, reader.position()); + } + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.STRING), column("bad", ColumnType.UUID), + column("c", ColumnType.VARCHAR), column("sym", ColumnType.SYMBOL)); + binding.longColumn("a", 10).stringColumn("a", "ignored"); + buffer.nextRow(); + binding.longColumn("sym", 7).longColumn("sym", Long.MIN_VALUE); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("bad", 20)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.longColumn("c", 30); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, QwpConstants.TYPE_VARCHAR, 3, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.byteValue()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(6, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(2, reader.intValue()); + Assert.assertArrayEquals(new byte[]{'1', '0'}, reader.bytes(2)); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(5, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(2, reader.intValue()); + Assert.assertArrayEquals(new byte[]{'3', '0'}, reader.bytes(2)); + Assert.assertEquals(size, reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.longColumn("value", 8); + buffer.nextRow(); + buffer.reset(); + binding.longColumn("value", 42); + buffer.nextRow(); + binding.longColumn("value", 43); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reset.byteValue()); + Assert.assertEquals(2, reset.varint()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reset.stringBytes()); + Assert.assertArrayEquals(new byte[]{'4', '3'}, reset.stringBytes()); + Assert.assertEquals(0, reset.varint()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(0, reset.varint()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingSchemaInfersNativeLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.longColumn("value", 42); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + Assert.assertEquals(QwpConstants.TYPE_LONG, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(42, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testOwnerBackedSymbolUsesGlobalDictionary() throws Exception { + assertMemoryLeak(() -> { + TestWebSocketServer.WebSocketServerHandler handler = new TestWebSocketServer.WebSocketServerHandler() { }; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender senderApi = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + QwpWebSocketSender sender = (QwpWebSocketSender) senderApi; + Assert.assertEquals(0, sender.getOrAddGlobalSymbol("prefix")); + GlobalSymbolDictionary mirror = new GlobalSymbolDictionary(); + Assert.assertEquals(0, mirror.getOrAddSymbol("prefix")); + try (QwpTableBuffer buffer = new QwpTableBuffer("t", sender); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.longColumn("value", 42); + buffer.nextRow(); + binding.longColumn("value", 43); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("42")); + Assert.assertEquals(2, sender.getOrAddGlobalSymbol("43")); + Assert.assertEquals(0, buffer.getColumn(0).getSymbolDictionarySize()); + mirror.getOrAddSymbol("42"); + mirror.getOrAddSymbol("43"); + encoder.beginSchemaMessage(1, mirror, -1, 2); + encoder.addSchemaTable(buffer, binding.getTableId(), binding.getMetadataVersion()); + int size = encoder.finishMessage(); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(3, reader.varint()); + Assert.assertArrayEquals("prefix".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("42".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("43".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + } + }); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, response((byte) QwpSchemaProtocol.RESULT_KNOWN, columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static byte[] frame(byte[] payload) { + return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload).array(); + } + + private static QwpSchemaResponse missing() { + ByteBuffer payload = ByteBuffer.allocate(1 + 8 + 1).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING); + return decodeResponse(payload.array()); + } + + private static QwpSchemaResponse response(byte result, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) length += column.length; + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put(result) + .putInt(result == QwpSchemaProtocol.RESULT_KNOWN ? 1 : -1) + .putLong(result == QwpSchemaProtocol.RESULT_KNOWN ? 1 : -1) + .putShort((short) -1).putShort((short) columns.length); + for (byte[] column : columns) payload.put(column); + return decodeResponse(payload.array()); + } + + private static QwpSchemaResponse decodeResponse(byte[] payload) { + byte[] frame = frame(payload); + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) Unsafe.getUnsafe().putByte(address + i, frame[i]); + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + return tableReader(encoder, size, wireType, rows, 1); + } + + private static void skipTableHeader(Reader reader, byte wireType, int rows) { + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.byteValue()); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + if (columns == 1) { + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.byteValue()); + } + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) return ColumnType.STRING; + if ("VARCHAR".equals(value)) return ColumnType.VARCHAR; + if ("SYMBOL".equals(value)) return ColumnType.SYMBOL; + throw new AssertionError(value); + } + + private static byte wireType(String value) { + return "SYMBOL".equals(value) ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR; + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(reason, error.getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int byteValue() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private byte[] bytes(int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) value[i] = (byte) byteValue(); + return value; + } + + private int intValue() { + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long longValue() { + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + private int position() { return position; } + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + private String string() { return new String(stringBytes(), StandardCharsets.UTF_8); } + private byte[] stringBytes() { return bytes(varint()); } + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = byteValue(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java new file mode 100644 index 000000000..6ae74134a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java @@ -0,0 +1,304 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongTimestampTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv"; + + @Test + public void testCorpusUsesExactTargetUnitWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongTimestampTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + int target = targetType(fields[2]); + byte wireType = wireType(fields[3]); + long value = Long.parseLong(fields[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.longColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, "value", wireType); + if ("".equals(fields[4])) { + Assert.assertEquals("true", fields[5]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals("false", fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(12, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + byte wireType = target == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.longColumn("value", Long.MIN_VALUE).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.longColumn("value", Long.MIN_VALUE + 1); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 3, "value", wireType); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.MIN_VALUE + 1, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.longColumn("value", Long.MAX_VALUE); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableReader(encoder, resetSize, 1, "value", wireType); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(Long.MAX_VALUE, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.TIMESTAMP_MICRO), + column("bad", ColumnType.UUID), + column("c", ColumnType.TIMESTAMP_NANO)); + binding.longColumn("a", -1); + buffer.nextRow(); + binding.longColumn("a", 2); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("bad", 3)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.longColumn("c", 4); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNamedDesignatedRejection() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.longColumn("value", Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.longColumn("ts", 1)); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + Reader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static int targetType(String value) { + return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; + } + + private static byte wireType(String value) { + return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java new file mode 100644 index 000000000..2ee58d1a7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java @@ -0,0 +1,494 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingNativeDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv"; + private static final String HEADER = "# case_id\tsource_type\tsource_scale\tsource_ll_hex\tsource_lh_hex\tsource_hl_hex\tsource_hh_hex\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWireAndPreservesCaller() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingNativeDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + Assert.assertFalse(line.isEmpty()); + String[] f = line.split("\t", -1); + Assert.assertEquals(line, 16, f.length); + Assert.assertTrue("VALUE".equals(f[10]) || "INVALID".equals(f[10])); + assertTargetTag(f[7], Integer.parseInt(f[8])); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", ColumnType.getDecimalType(Integer.parseInt(f[8]), Integer.parseInt(f[9])))); + Object value = decimal(f); + long[] before = limbs(value); + int scale = scale(value); + if ("INVALID".equals(f[10])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> append(binding, value)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(0, buffer.getColumnCount()); + } else { + append(binding, value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, wireType(f[7])); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(f[9]), reader.u8()); + assertExpectedLimbs(reader, f); + Assert.assertEquals(size, reader.position()); + } + Assert.assertEquals(scale, scale(value)); + Assert.assertArrayEquals(before, limbs(value)); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(60, count); + }); + } + + @Test + public void testDuplicateRollbackResetOmissionAndRetainedTargetScale() throws Exception { + assertMemoryLeak(() -> { + int decimal = ColumnType.getDecimalType(18, 4); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", decimal), column("bad", ColumnType.getDecimalType(2, 0)), column("c", decimal)); + binding.decimalColumn("a", new Decimal64(15, 1)).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.decimalColumn("a", new Decimal64(2, 0)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.decimalColumn("bad", new Decimal128(0, 1000, 0))); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.decimalColumn("c", new Decimal256(0, 0, 0, 25, 2)); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 2, 2); + decimalDefinition(reader, "a", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reader, "c", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(4, reader.u8()); + Assert.assertEquals(15000, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(4, reader.u8()); + Assert.assertEquals(2500, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.decimalColumn("c", new Decimal64(3, 0)); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 2); + decimalDefinition(reset, "a", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reset, "c", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(4, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(4, reset.u8()); + Assert.assertEquals(30000, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingInfersNativeDecimalWidthAndScale() throws Exception { + assertMemoryLeak(() -> { + Object[] values = { + new Decimal64(12, 1), + new Decimal128(3, 4, 2), + new Decimal256(5, 6, 7, 8, 3) + }; + byte[] types = {QwpConstants.TYPE_DECIMAL64, QwpConstants.TYPE_DECIMAL128, QwpConstants.TYPE_DECIMAL256}; + for (int i = 0; i < values.length; i++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + append(binding, values[i]); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = unknownTableReader(encoder, size, types[i]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(scale(values[i]), reader.u8()); + for (long limb : wireLimbs(values[i])) { + Assert.assertEquals(limb, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testMalformedParameterizedAndDesignatedTargetsRemainRejected() throws Exception { + assertMemoryLeak(() -> { + Decimal64 value = new Decimal64(1, 0); + int[] malformed = { + ColumnType.DECIMAL64, + ColumnType.getDecimalType(18, 4) | 0x10000, + ColumnType.getDecimalType(18, 4) | 0x40000000 + }; + for (int type : malformed) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", type)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, + column("value", ColumnType.getDecimalType(18, 4))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + parameterizedColumn("value", ColumnType.getDecimalType(18, 4))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + }); + } + + @Test + public void testLegacyBufferStillChoosesScaleFromFirstValueInEachBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn( + "value", QwpConstants.TYPE_DECIMAL64, true); + column.addDecimal64(new Decimal64(12, 2)); + buffer.nextRow(); + int firstSize = encoder.encode(buffer); + Reader first = legacyTableReader(encoder, firstSize); + Assert.assertEquals(0, first.u8()); + Assert.assertEquals(2, first.u8()); + Assert.assertEquals(12, first.i64()); + Assert.assertEquals(firstSize, first.position()); + + buffer.reset(); + column.addDecimal64(new Decimal64(34, 3)); + buffer.nextRow(); + int secondSize = encoder.encode(buffer); + Reader second = legacyTableReader(encoder, secondSize); + Assert.assertEquals(0, second.u8()); + Assert.assertEquals(3, second.u8()); + Assert.assertEquals(34, second.i64()); + Assert.assertEquals(secondSize, second.position()); + } + }); + } + + private static void append(QwpSchemaBinding binding, Object value) { + if (value instanceof Decimal64) { + binding.decimalColumn("value", (Decimal64) value); + } else if (value instanceof Decimal128) { + binding.decimalColumn("value", (Decimal128) value); + } else { + binding.decimalColumn("value", (Decimal256) value); + } + } + + private static void assertExpectedLimbs(Reader reader, String[] f) { + int count = wireLongCount(f[7]); + for (int i = 0; i < count; i++) { + Assert.assertEquals(parseHex(f[11 + i]), reader.i64()); + } + for (int i = count; i < 4; i++) { + Assert.assertEquals("-", f[11 + i]); + } + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static void assertTargetTag(String name, int precision) { + String expected; + if (precision <= 2) { + expected = "DECIMAL8"; + } else if (precision <= 4) { + expected = "DECIMAL16"; + } else if (precision <= 9) { + expected = "DECIMAL32"; + } else if (precision <= 18) { + expected = "DECIMAL64"; + } else if (precision <= 38) { + expected = "DECIMAL128"; + } else { + expected = "DECIMAL256"; + } + Assert.assertEquals(expected, name); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static Object decimal(String[] f) { + int scale = Integer.parseInt(f[2]); + long ll = parseHex(f[3]); + switch (f[1]) { + case "DECIMAL64": + return new Decimal64(ll, scale); + case "DECIMAL128": + Decimal128 decimal = new Decimal128(); + decimal.of(parseHex(f[4]), ll, scale); + return decimal; + case "DECIMAL256": + return new Decimal256(parseHex(f[6]), parseHex(f[5]), parseHex(f[4]), ll, scale); + default: + throw new AssertionError("unknown source type " + f[1]); + } + } + + private static void decimalDefinition(Reader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static long[] limbs(Object value) { + if (value instanceof Decimal64) { + return new long[]{((Decimal64) value).getValue()}; + } + if (value instanceof Decimal128) { + return new long[]{((Decimal128) value).getHigh(), ((Decimal128) value).getLow()}; + } + Decimal256 decimal = (Decimal256) value; + return new long[]{decimal.getHh(), decimal.getHl(), decimal.getLh(), decimal.getLl()}; + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static Reader legacyTableReader(QwpWebSocketEncoder encoder, int size) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + return reader; + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static byte[] parameterizedColumn(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6 + 4).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 4).putInt(1).array(); + } + + private static int scale(Object value) { + if (value instanceof Decimal64) { + return ((Decimal64) value).getScale(); + } + if (value instanceof Decimal128) { + return ((Decimal128) value).getScale(); + } + return ((Decimal256) value).getScale(); + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + Reader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static Reader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static long[] wireLimbs(Object value) { + if (value instanceof Decimal64) { + return new long[]{((Decimal64) value).getValue()}; + } + if (value instanceof Decimal128) { + Decimal128 decimal = (Decimal128) value; + return new long[]{decimal.getLow(), decimal.getHigh()}; + } + Decimal256 decimal = (Decimal256) value; + return new long[]{decimal.getLl(), decimal.getLh(), decimal.getHl(), decimal.getHh()}; + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java new file mode 100644 index 000000000..4a6599064 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java @@ -0,0 +1,342 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-decimal.tsv"; + private static final String HEADER = "# case_id\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[] targets = new boolean[6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 11, fields.length); + Vector vector = new Vector(fields); + targets[targetIndex(vector.targetType)] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + if (vector.invalid()) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", vector.input)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } else { + binding.stringColumn("value", vector.input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); + Reader reader = tableReader(encoder, size, wireType(vector.targetType)); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(55, count); + for (int i = 0; i < targets.length; i++) { + Assert.assertTrue("missing target " + i, targets[i]); + } + }); + } + + @Test + public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(3, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + binding.stringColumn("value", "1.0").stringColumn("value", "invalid"); + buffer.nextRow(); + + binding.stringColumn("only_b", "2.0"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", "100.0")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.stringColumn("value", "-2.5"); + buffer.nextRow(); + binding.stringColumn("value", null).stringColumn("value", "invalid"); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 4, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0c, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-25, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 1); + decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static void decimalDefinition(Reader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static QwpSchemaResponse known(byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) -1).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": return 0; + case "DECIMAL16": return 1; + case "DECIMAL32": return 2; + case "DECIMAL64": return 3; + case "DECIMAL128": return 4; + default: return 5; + } + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + Reader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) return QwpConstants.TYPE_DECIMAL128; + if ("DECIMAL256".equals(target)) return QwpConstants.TYPE_DECIMAL256; + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) return 2; + if ("DECIMAL256".equals(target)) return 4; + return 1; + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) bytes[i] = (byte) u8(); + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final String input; + private final String outcome; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + input = "".equals(fields[1]) ? null : "".equals(fields[1]) ? "" : fields[1]; + targetType = fields[2]; + targetPrecision = Integer.parseInt(fields[3]); + targetScale = Integer.parseInt(fields[4]); + outcome = fields[5]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) expectedLimbs[i] = parseHex(fields[6 + i]); + for (int i = limbs; i < 4; i++) Assert.assertEquals("-", fields[6 + i]); + if (outcome.equals("INVALID")) { + Assert.assertEquals("-", fields[10]); + } else if (outcome.equals("NULL")) { + Assert.assertEquals("NULL", fields[10]); + } else { + Assert.assertNotEquals("-", fields[10]); + Assert.assertNotEquals("NULL", fields[10]); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) return "DECIMAL8"; + if (precision <= 4) return "DECIMAL16"; + if (precision <= 9) return "DECIMAL32"; + if (precision <= 18) return "DECIMAL64"; + if (precision <= 38) return "DECIMAL128"; + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java new file mode 100644 index 000000000..d166a1c01 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java @@ -0,0 +1,365 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringGeoHashTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv"; + private static final String HEADER = "# case_id\tinput\tbits\toutcome\tvalue_hex"; + + @Test + public void testCorpusUsesExactGeoHashWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean headerSeen = false; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse(headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + int bits = Integer.parseInt(fields[2]); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", ColumnType.getGeoHashTypeWithBits(bits)))); + CharSequence value = "".equals(fields[1]) ? null + : "".equals(fields[1]) ? "" : fields[1]; + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + } else if ("VALUE".equals(fields[3]) || "NULL".equals(fields[3])) { + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bits, reader.varint()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(bits, reader.varint()); + long expected = Long.parseUnsignedLong(fields[4], 16); + for (int i = 0; i < (bits + 7) / 8; i++) { + Assert.assertEquals((int) ((expected >>> (8 * i)) & 0xff), reader.u8()); + } + } + Assert.assertEquals(size, reader.position); + } else { + throw new AssertionError("unknown outcome: " + fields[3]); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue(headerSeen); + Assert.assertEquals(18, count); + }); + } + + @Test + public void testDuplicateBitmapRollbackResetInferenceAndGuards() throws Exception { + assertMemoryLeak(() -> { + int geo8 = ColumnType.getGeoHashTypeWithBits(8); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("a", geo8), column("bad", ColumnType.LONG), column("c", geo8))); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "zz"); + buffer.nextRow(); + binding.stringColumn("a", ""); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "zz"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "z")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "04"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 5, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + assertGeo(reader, 29, 8, 0xff); + assertGeo(reader, 15, 8, 1); + Assert.assertEquals(size, reader.position); + + buffer.reset(); + binding.stringColumn("a", "04"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); + assertGeo(reset, 0, 8, 1); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(8, reset.varint()); + Assert.assertEquals(resetSize, reset.position); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "z"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.unsupportedColumn("value", "GEOHASH")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, parameterizedColumn("value", geo8))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "z")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "z")); + } + }); + } + + @Test + public void testMalformedPackedGeoHashTargetsRemainTypedUnsupported() throws Exception { + assertMemoryLeak(() -> { + int geoFlag = 1 << 16; + int[] malformed = { + ColumnType.GEOHASH, + geoFlag | ColumnType.GEOBYTE, + geoFlag | (61 << 8) | ColumnType.GEOLONG, + geoFlag | (8 << 8) | ColumnType.STRING, + ColumnType.getGeoHashTypeWithBits(8) | (1 << 24) + }; + for (int targetType : malformed) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", targetType))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "zz")); + } + } + }); + } + + @Test + public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exception { + assertMemoryLeak(() -> { + for (int bits = 1; bits <= 60; bits++) { + int targetType = ColumnType.getGeoHashTypeWithBits(bits); + StringBuilder input = new StringBuilder((bits + 4) / 5); + for (int i = 0; i < (bits + 4) / 5; i++) { + input.append('z'); + } + long max = (1L << bits) - 1; + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", targetType))); + binding.stringColumn("value", input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1); + assertGeo(reader, 0, bits, max); + Assert.assertEquals(size, reader.position); + + buffer.reset(); + binding.stringColumn("value", input); + buffer.nextRow(); + binding.stringColumn("value", null); + buffer.nextRow(); + binding.stringColumn("value", ""); + buffer.nextRow(); + size = encoder.encodeSchema(buffer); + reader = tableReader(encoder, size, 3); + assertGeo(reader, 6, bits, max); + Assert.assertEquals(size, reader.position); + } + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + buffer.getOrCreateColumn("value", QwpConstants.TYPE_GEOHASH, true) + .addGeoHash(0x0fffffffffffffffL, 60); + buffer.nextRow(); + int size = encoder.encode(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(60, reader.varint()); + Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); + Assert.assertEquals(size, reader.position); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + buffer.getOrCreateColumn("value", QwpConstants.TYPE_GEOHASH, false) + .addGeoHash(0x0fffffffffffffffL, 60); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer, 1, 1); + Reader reader = tableReader(encoder, size, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(60, reader.varint()); + Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); + Assert.assertEquals(size, reader.position); + } + }); + } + + private static void assertGeo(Reader reader, int bitmap, int bits, long value) { + Assert.assertEquals(bitmap < 0 ? 0 : 1, reader.u8()); + if (bitmap >= 0) { + Assert.assertEquals(bitmap, reader.u8()); + } + Assert.assertEquals(bits, reader.varint()); + for (int i = 0; i < (bits + 7) / 8; i++) { + Assert.assertEquals((int) ((value >>> (8 * i)) & 0xff), reader.u8()); + } + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] parameterizedColumn(String name, int type) { + return column(name, type, new byte[]{1}); + } + + private static byte[] column(String name, int type, byte[] parameters) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6 + parameters.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) parameters.length) + .put(parameters).array(); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + Reader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + return reader; + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { this.address = address; this.limit = limit; } + private int i32() { check(4); int value = Unsafe.getUnsafe().getInt(address + position); position += 4; return value; } + private long i64() { check(8); long value = Unsafe.getUnsafe().getLong(address + position); position += 8; return value; } + private void skip(int length) { check(length); position += length; } + private String string() { int length = varint(); byte[] bytes = new byte[length]; for (int i = 0; i < length; i++) bytes[i] = (byte) u8(); return new String(bytes, StandardCharsets.UTF_8); } + private int u8() { check(1); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } + private int u16() { check(2); int value = Unsafe.getUnsafe().getShort(address + position) & 0xffff; position += 2; return value; } + private int varint() { int result = 0; int shift = 0; int b; do { b = u8(); result |= (b & 0x7f) << shift; shift += 7; } while ((b & 0x80) != 0); return result; } + private void check(int size) { Assert.assertTrue(position + size <= limit); } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java new file mode 100644 index 000000000..a34216733 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java @@ -0,0 +1,309 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringLong256Test { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv"; + private static final String HEADER = "# case_id\tinput\toutcome\tl0_hex\tl1_hex\tl2_hex\tl3_hex"; + + @Test + public void testCorpusUsesExactLong256Wire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringLong256Test.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + boolean headerSeen = false; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.LONG256)); + if ("VALUE".equals(fields[2])) { + binding.stringColumn("value", fields[1]); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1); + Assert.assertEquals(0, reader.u8()); + for (int i = 3; i < 7; i++) { + Assert.assertEquals(parseHexLong(fields[i]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } else if ("NULL".equals(fields[2])) { + Assert.assertEquals("", fields[1]); + binding.stringColumn("value", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } else if ("INVALID".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", fields[1])); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(0, buffer.getColumnDefs().length); + } else { + throw new AssertionError("unknown outcome: " + fields[2]); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(23, count); + }); + } + + @Test + public void testDuplicateNullOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.LONG256), column("bad", ColumnType.LONG), column("c", ColumnType.LONG256)); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "0x01").stringColumn("a", "invalid duplicate"); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "0x02"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "0x03")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "0x04"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(4, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + assertLong256Values(reader, 13, 1); + assertLong256Values(reader, 7, 4); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "0x05"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(5, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingStringInferenceRemainsVarchar() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "0x00"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + } + + @Test + public void testParameterizedDesignatedAndOtherTargetsRemainUnsupported() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + parameterizedColumn("value", ColumnType.LONG256), column("other", ColumnType.UUID)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "0x00")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("other", "0x00")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + response(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "0x00")); + } + } + + private static void assertLong256Values(Reader reader, int bitmap, long value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(value, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, response(columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] parameterizedColumn(String name, int type) { + return column(name, type, new byte[]{1}); + } + + private static byte[] column(String name, int type, byte[] parameters) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6 + parameters.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) parameters.length) + .put(parameters).array(); + } + + private static QwpSchemaResponse missing() { + ByteBuffer payload = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING); + return decode(payload.array()); + } + + private static QwpSchemaResponse response(byte[]... columns) { + return response(-1, columns); + } + + private static QwpSchemaResponse response(int designatedIndex, byte[]... columns) { + int length = 26; + for (byte[] column : columns) length += column.length; + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) payload.put(column); + return decode(payload.array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + return tableReader(encoder, size, rows, "value"); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(column, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + return reader; + } + + private static long parseHexLong(String value) { + return Long.parseUnsignedLong(value, 16); + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(reason, error.getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private String ascii(int length) { return new String(bytes(length), StandardCharsets.UTF_8); } + private byte[] bytes(int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) value[i] = (byte) u8(); + return value; + } + private int i32() { int value = Unsafe.getUnsafe().getInt(address + position); position += 4; return value; } + private long i64() { long value = Unsafe.getUnsafe().getLong(address + position); position += 8; return value; } + private int position() { return position; } + private void skip(int length) { position += length; Assert.assertTrue(position <= limit); } + private String string() { return ascii(varint()); } + private int u8() { Assert.assertTrue(position < limit); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } + private int varint() { + int result = 0; + int shift = 0; + int value; + do { value = u8(); result |= (value & 0x7f) << shift; shift += 7; } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java new file mode 100644 index 000000000..dc848d7e7 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java @@ -0,0 +1,408 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringNumericTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-numeric.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringNumericTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] f = line.split("\\t", -1); + Assert.assertEquals(line, 7, f.length); + Assert.assertTrue(line, "VALID".equals(f[6]) || "INVALID".equals(f[6])); + CharSequence value = "".equals(f[1]) ? null : utf16(f[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(f[2]))); + if ("INVALID".equals(f[6])) { + Assert.assertEquals(f[0], "", f[4]); + Assert.assertEquals(f[0], "", f[5]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("value", value)); + } else { + binding.stringColumn("value", value); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[3]), 1, 1); + if ("".equals(f[4])) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + } else { + Assert.assertEquals(0, reader.byteValue()); + byte[] expected = littleEndianHex(f[4]); + Assert.assertArrayEquals(f[0], expected, reader.bytes(expected.length)); + } + Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError | RuntimeException e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(429, count); + }); + } + + @Test + public void testDuplicatePrecedesInvalidParseAndUnsupportedConversion() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("n", ColumnType.INT), column("uuid", ColumnType.UUID)); + binding.stringColumn("n", "42").stringColumn("n", "invalid"); + binding.uuidColumn("uuid", 1, 2).stringColumn("uuid", "not-a-uuid"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 1, 2); + Assert.assertEquals("n", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.byteValue()); + Assert.assertEquals("uuid", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(42, reader.intValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.TIMESTAMP_MICRO)); + binding.timestampColumn("value", 123, ChronoUnit.MICROS) + .stringColumn("value", "not-a-timestamp-conversion"); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_TIMESTAMP, 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(123, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.INT), column("only_b", ColumnType.LONG), + column("bad", ColumnType.SHORT), column("c", ColumnType.DOUBLE)); + binding.stringColumn("a", "1"); + buffer.nextRow(); + binding.stringColumn("only_b", "2"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("bad", "+3")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "-0.0"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.byteValue()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0x8000000000000000L, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testResetClearAndMutableInput() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.LONG)); + StringBuilder mutable = new StringBuilder("7"); + binding.stringColumn("value", mutable); + mutable.replace(0, 1, "9"); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(7, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + + buffer.reset(); + binding.stringColumn("value", "8"); + buffer.nextRow(); + reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(8, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + assertIllegalState(() -> binding.stringColumn("value", "1")); + } + }); + } + + @Test + public void testNullAndParsedSentinelsHaveDistinctWirePresence() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.INT)); + binding.stringColumn("value", "-2147483648"); + buffer.nextRow(); + binding.stringColumn("value", null); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_INT, 2, 1); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(Integer.MIN_VALUE, reader.intValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("bool", ColumnType.BOOLEAN), column("date", ColumnType.DATE), + column("future", ColumnType.INT, new byte[]{1}), + column("flagged", ColumnType.LONG | 0x10000)); + assertSchemaError(LineSenderSchemaException.Reason.INVALID_VALUE, "bool", "BOOLEAN", () -> binding.stringColumn("bool", "2")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "date", "DATE", () -> binding.stringColumn("date", "1")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "future", "INT", () -> binding.stringColumn("future", "1")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "flagged", "unknown", () -> binding.stringColumn("flagged", "1")); + rollback(buffer); + binding.stringColumn("missing", "1"); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.DOUBLE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("d", null)); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); + } + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN).putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(length).put(payload.array()); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.byteValue()); + return reader; + } + + private static void skipTablePrefix(Reader reader, int rows, int columns) { + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + } + + private static int targetType(String value) { + switch (value) { + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError(value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError(value); + } + } + + private static String utf16(String hex) { + Assert.assertEquals(0, hex.length() & 3); + char[] chars = new char[hex.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static byte[] littleEndianHex(String hex) { + Assert.assertEquals(0, hex.length() & 1); + byte[] bytes = new byte[hex.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring((bytes.length - 1 - i) * 2, (bytes.length - i) * 2), 16); + } + return bytes; + } + + private static void rollback(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static void assertIllegalState(Runnable runnable) { + Assert.assertThrows(IllegalStateException.class, runnable::run); + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable runnable) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(reason, error.getReason()); + Assert.assertFalse(error.isRetryable()); + } + + private static void assertSchemaError( + LineSenderSchemaException.Reason reason, + String column, + String target, + Runnable runnable + ) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("table=t")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=" + column)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=STRING")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + target)); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int byteValue() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 255; + } + + private byte[] bytes(int count) { + Assert.assertTrue(position + count <= limit); + byte[] bytes = new byte[count]; + for (int i = 0; i < count; i++) { + bytes[i] = (byte) byteValue(); + } + return bytes; + } + + private int intValue() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long longValue() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int count) { + Assert.assertTrue(position + count <= limit); + position += count; + } + + private String string() { + return new String(bytes(varint()), StandardCharsets.UTF_8); + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = byteValue(); + result |= (value & 127) << shift; + shift += 7; + } while ((value & 128) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java new file mode 100644 index 000000000..6d2428e8d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java @@ -0,0 +1,341 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringTimestampTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv"; + + @Test + public void testCorpusUsesExactTargetUnitWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringTimestampTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("case_id\ttarget\tinput_kind\tinput\toutcome\texpected_raw\tfeature", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertTrue(line, "VALID".equals(fields[4]) || "NULL".equals(fields[4]) || "INVALID".equals(fields[4])); + int target = targetType(fields[1]); + byte wireType = wireType(fields[1]); + CharSequence value = input(fields[2], fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + if ("INVALID".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + Assert.assertEquals(0, buffer.getRowCount()); + count++; + continue; + } + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, "value", wireType); + if ("NULL".equals(fields[4])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(110, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + byte wireType = target == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.stringColumn("value", null).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("value", "1970-01-01T00:00:00.000001Z"); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 3, "value", wireType); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? 1 : 1000, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("value", "1969-12-31T23:59:59.999999Z"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableReader(encoder, resetSize, 1, "value", wireType); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.TIMESTAMP_MICRO), + column("bad", ColumnType.UUID), + column("c", ColumnType.TIMESTAMP_NANO)); + binding.stringColumn("a", "1969-12-31T23:59:59.999999Z"); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-01T00:00:00.000002Z"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "1970-01-01T00:00:00.000004Z"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4000, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNamedDesignatedRejection() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "1970-01-01T00:00:00Z"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(20, reader.i32()); + Assert.assertEquals("1970-01-01T00:00:00Z", reader.ascii(20)); + Assert.assertEquals(size, reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "1970-01-01T00:00:00Z")); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + Reader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static int targetType(String value) { + Assert.assertTrue(value, "TIMESTAMP".equals(value) || "TIMESTAMP_NANOS".equals(value)); + return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; + } + + private static CharSequence input(String kind, String value) { + Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); + if ("".equals(value)) { + return null; + } + if ("TEXT".equals(kind)) { + return value; + } + String[] units = value.split(","); + char[] chars = new char[units.length]; + for (int i = 0; i < units.length; i++) { + chars[i] = (char) Integer.parseInt(units[i], 16); + } + return new String(chars); + } + + private static byte wireType(String value) { + return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position <= limit - Integer.BYTES); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private String ascii(int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = (byte) u8(); + } + return new String(value, StandardCharsets.UTF_8); + } + + private long i64() { + Assert.assertTrue(position <= limit - Long.BYTES); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java new file mode 100644 index 000000000..3da239ea0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -0,0 +1,1284 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingTest { + @Test + public void testTimestampInputConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try { + int targetType = timestampColumnType(fields[4]); + byte wireType = timestampWireType(fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + Runnable write; + if ("LONG".equals(fields[1])) { + long input = Long.parseLong(fields[2]); + ChronoUnit unit = ChronoUnit.valueOf(fields[3]); + write = () -> rows.timestampColumn("ts", input, unit); + } else if ("INSTANT".equals(fields[1])) { + Instant input = Instant.parse(fields[2]); + Assert.assertEquals("-", fields[3]); + write = () -> rows.timestampColumn("ts", input); + } else { + throw new AssertionError("unknown input_kind: " + fields[1]); + } + if ("".equals(fields[5])) { + Assert.assertEquals("", fields[6]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, write); + continue; + } + write.run(); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.byteValue()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.byteValue()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + } + } + + @Test + public void testTimestampOverloadValidationAndDuplicateSuppression() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("ts", Instant.ofEpochSecond(0, 123)) + .timestampColumn("ts", (Instant) null) + .timestampColumn("ts", Long.MAX_VALUE, ChronoUnit.WEEKS); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, + QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(123, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", (Instant) null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.WEEKS)); + } + for (int targetType : new int[]{ColumnType.LONG, ColumnType.TIMESTAMP_NANO | 0x10000}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", (Instant) null)); + } + } + }); + } + + @Test + public void testTimestampInstantFailureRollsBackOnlyCurrentRowColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("late", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("micro", Instant.ofEpochSecond(1)); + buffer.nextRow(); + rows.timestampColumn("micro", Instant.ofEpochSecond(2)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("late", Instant.MAX)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.timestampColumn("micro", Instant.ofEpochSecond(3)) + .timestampColumn("late", Instant.ofEpochSecond(0, 4)); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1_000_000, reader.longValue()); + Assert.assertEquals(3_000_000, reader.longValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(4, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testTimestampUnitConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/timestamp-units.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try { + long input = Long.parseLong(fields[1]); + ChronoUnit unit = ChronoUnit.valueOf(fields[2]); + int targetType = timestampColumnType(fields[3]); + byte wireType = timestampWireType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + if ("".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", input, unit)); + continue; + } + rows.timestampColumn("ts", input, unit); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.byteValue()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.byteValue()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + } + } + + @Test + public void testTimestampValidationDuplicateAndRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("nano", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("micro", 7, ChronoUnit.MICROS) + .timestampColumn("micro", Long.MAX_VALUE, null) + .timestampColumn("micro", Long.MAX_VALUE, ChronoUnit.SECONDS); + buffer.nextRow(); + + rows.timestampColumn("micro", 8, ChronoUnit.MICROS); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("nano", Long.MAX_VALUE, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + rows.timestampColumn("micro", 9, ChronoUnit.MICROS) + .timestampColumn("nano", 10, ChronoUnit.NANOS); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(7, reader.longValue()); + Assert.assertEquals(9, reader.longValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(10, reader.longValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testTimestampNullUnsupportedAndBindingLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("nano", ColumnType.TIMESTAMP_NANO)); + rows.stringColumn("micro", null).stringColumn("nano", null); + buffer.nextRow(); + rows.stringColumn("micro", null); + buffer.nextRow(); + Reader nulls = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(1, nulls.byteValue()); + Assert.assertEquals(3, nulls.byteValue()); + Assert.assertEquals(0, nulls.byteValue()); + Assert.assertEquals(1, nulls.byteValue()); + Assert.assertEquals(3, nulls.byteValue()); + Assert.assertEquals(0, nulls.byteValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), nulls.position()); + + buffer.reset(); + rows.timestampColumn("nano", 11, ChronoUnit.NANOS); + buffer.nextRow(); + buffer.clear(); + assertIllegalState(() -> rows.timestampColumn("nano", 12, ChronoUnit.NANOS)); + assertIllegalState(() -> rows.timestampColumn("nano", Instant.EPOCH)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("x", ColumnType.LONG)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("x", Long.MIN_VALUE, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.timestampColumn("missing", 1, ChronoUnit.MICROS); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", 1, null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.WEEKS)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + for (int targetType : new int[]{ColumnType.TIMESTAMP_MICRO | 0x10000, ColumnType.TIMESTAMP_NANO | 0x10000}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", Instant.EPOCH)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.stringColumn("ts", null)); + } + }); + } + @Test + public void testClassifiesSchemaLessResponses() throws Exception { + assertCtorReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "access denied"); + assertCtorReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "temporarily unavailable"); + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)).longColumn("x", 1); + } + assertCtorReason(QwpSchemaProtocol.RESULT_TOO_LARGE, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "response limit"); + } + + @Test + public void testInfersNativeTargetsForMissingSchemaAndColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.longColumn("l", 1) + .stringColumn("s", "x") + .symbol("sym", "y") + .boolColumn("b", true) + .floatColumn("f", 1) + .doubleColumn("d", 2) + .uuidColumn("u", 3, 4) + .binaryColumn("bin", new byte[]{5}) + .timestampColumn("tm", 6, ChronoUnit.MICROS) + .timestampColumn("tn", 7, ChronoUnit.NANOS) + .timestampColumn("ti", Instant.EPOCH); + buffer.nextRow(); + byte[] expected = { + QwpConstants.TYPE_LONG, QwpConstants.TYPE_VARCHAR, QwpConstants.TYPE_SYMBOL, + QwpConstants.TYPE_BOOLEAN, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE, + QwpConstants.TYPE_UUID, QwpConstants.TYPE_BINARY, QwpConstants.TYPE_TIMESTAMP, + QwpConstants.TYPE_TIMESTAMP_NANOS, QwpConstants.TYPE_TIMESTAMP + }; + QwpColumnDef[] definitions = buffer.getColumnDefs(); + Assert.assertEquals(expected.length, definitions.length); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals("column " + i, expected[i], definitions[i].getTypeCode()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.uuidColumn("u", 0x0102030405060708L, 0x1112131415161718L) + .floatColumn("f", Float.intBitsToFloat(0x3fc00000)) + .stringColumn("s", "quest"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, -1, -1, 1, + QwpConstants.TYPE_UUID, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0x0102030405060708L, reader.longValue()); + Assert.assertEquals(0x1112131415161718L, reader.longValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0x3fc00000, reader.intValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(5, reader.intValue()); + Assert.assertEquals("quest", reader.ascii(5)); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("known", ColumnType.LONG)); + rows.longColumn("missing", 1) + .unsupportedColumn("missing", "INT") + .uuidColumn("missing", 2, 3); + buffer.nextRow(); + LineSenderSchemaException conflict = Assert.assertThrows( + LineSenderSchemaException.class, + () -> rows.uuidColumn("missing", 4, 5) + ); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, conflict.getReason()); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.longColumn("a", 1); + buffer.nextRow(); + try { + rows.stringColumn("failed_only", new CharSequence() { + @Override + public char charAt(int index) { + if (index == 1) { + throw new IllegalStateException("fixture failure"); + } + return 'x'; + } + + @Override + public int length() { + return 2; + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + }); + Assert.fail("expected fixture failure"); + } catch (IllegalStateException e) { + Assert.assertEquals("fixture failure", e.getMessage()); + } + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("a", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, -1, -1, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(3, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer micros = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(micros, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.designatedTimestamp(1, ChronoUnit.MICROS); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, micros.getColumnDefs()[0].getTypeCode()); + micros.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.designatedTimestamp(2, ChronoUnit.NANOS)); + micros.cancelCurrentRow(); + micros.rollbackUncommittedColumns(); + } + try (QwpTableBuffer nanos = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(nanos, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.designatedTimestamp(1, ChronoUnit.NANOS); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, nanos.getColumnDefs()[0].getTypeCode()); + } + }); + } + + @Test + public void testDuplicateUsesFirstValueAcrossInputSetters() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.uuidColumn("u", 1, 2).longColumn("u", 3); + rows.longColumn("l", 4).stringColumn("l", "not-a-long"); + buffer.nextRow(); + rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef") + .stringColumn("u", "malformed"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); + Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(4, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testEmptyAndIncompleteEncodeContracts() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("l", ColumnType.LONG)); + Assert.assertEquals(0, encoder.encodeSchema(buffer)); + rows.longColumn("l", 1); + assertIllegalState(() -> encoder.encodeSchema(buffer)); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 1); + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + buffer.reset(); + Assert.assertEquals(0, encoder.encodeSchema(buffer)); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + }); + } + + @Test + public void testLifecycleAndNameValidation() throws Exception { + try (QwpTableBuffer invalid = new QwpTableBuffer("bad\nname")) { + assertIllegalArgument(() -> new QwpSchemaBinding(invalid, known(column("x", ColumnType.LONG)))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("x", ColumnType.LONG)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("bad\nname", 1)); + rows.longColumn("missing", 1); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding stale = rows(buffer, column("x", ColumnType.LONG)); + buffer.clear(); + assertIllegalState(() -> stale.longColumn("x", 1)); + assertIllegalState(() -> stale.timestampColumn("x", Instant.EPOCH)); + } + } + + @Test + public void testLongNumericConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/long-to-numeric.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 4, fields.length); + try { + long input = Long.parseLong(fields[1]); + int targetType = numericColumnType(fields[2]); + byte wireType = numericWireType(fields[2]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", targetType)); + if ("".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.longColumn("n", input)); + continue; + } + rows.longColumn("n", input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, wireType); + if ("".equals(fields[3])) { + if (input == Long.MIN_VALUE) { + assertNumericNull(reader, fields[2]); + } else { + Assert.assertEquals("INT sentinel is carried as a non-null wire value", 0, reader.byteValue()); + Assert.assertEquals(Integer.MIN_VALUE, reader.intValue()); + } + } else { + Assert.assertEquals(0, reader.byteValue()); + assertNumericValue(reader, fields[2], fields[3]); + } + Assert.assertEquals(size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + } + } + + @Test + public void testLongNumericDuplicateFirstAndFailureRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("b", ColumnType.BYTE), column("i", ColumnType.INT)); + rows.longColumn("b", 7).longColumn("b", 128).stringColumn("b", "unsupported"); + rows.longColumn("i", 1); + buffer.nextRow(); + rows.longColumn("i", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("b", 128)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("b", 3).longColumn("i", 4); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(7, reader.byteValue()); + Assert.assertEquals(3, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(4, reader.intValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLongRangeDiagnosticIsReadableAndDoesNotEchoValue() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("b", ColumnType.BYTE)); + try { + rows.longColumn("b", 9_876_543_210L); + Assert.fail("expected range failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + Assert.assertFalse(e.isRetryable()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("table=t")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("column=b")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("inputType=LONG")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("targetType=BYTE(" + ColumnType.BYTE + ")")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("outside target type range")); + Assert.assertFalse(e.getMessage(), e.getMessage().contains("9876543210")); + } + } + } + + @Test + public void testLongNullRejectsUuidUnlessDuplicate() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.longColumn("u", Long.MIN_VALUE)); + rows.uuidColumn("u", 1, 2).longColumn("u", Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLongNumericDuplicateFirstAcrossEveryTarget() throws Exception { + assertMemoryLeak(() -> { + byte[][] schema = { + column("b", ColumnType.BYTE), column("s", ColumnType.SHORT), + column("i", ColumnType.INT), column("l", ColumnType.LONG), + column("f", ColumnType.FLOAT), column("d", ColumnType.DOUBLE)}; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, schema); + String[] names = {"b", "s", "i", "l", "f", "d"}; + for (String name : names) { + rows.longColumn(name, 7).stringColumn(name, "not-numeric").uuidColumn(name, 1, 2); + } + rows.longColumn("b", 128).longColumn("s", 32768).longColumn("i", 2147483648L); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.shortValue()); + Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.intValue()); + Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.longValue()); + Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(Float.floatToRawIntBits(7), reader.intValue()); + Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(Double.doubleToRawLongBits(7), reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNumericFailureRemovesColumnsIntroducedOnlyInFailedRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("a", ColumnType.LONG), column("only_b", ColumnType.BYTE), + column("bad", ColumnType.SHORT)); + rows.longColumn("a", 1); + buffer.nextRow(); + rows.longColumn("only_b", 5); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("bad", 32768)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("a", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(3, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNumericExtensionRejectedOnlyWhenUsed() throws Exception { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("future_float", ColumnType.FLOAT, new byte[]{1}), column("plain", ColumnType.BYTE)); + rows.longColumn("plain", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.longColumn("future_float", 2)); + } + } + + @Test + public void testNullStringSupportsAllNumericTargets() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("b", ColumnType.BYTE), column("s", ColumnType.SHORT), + column("i", ColumnType.INT), column("l", ColumnType.LONG), + column("f", ColumnType.FLOAT), column("d", ColumnType.DOUBLE)); + rows.stringColumn("b", null).stringColumn("s", null).stringColumn("i", null) + .stringColumn("l", null).stringColumn("f", null).stringColumn("d", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); + assertNumericNull(reader, "BYTE"); + assertNumericNull(reader, "SHORT"); + assertNumericNull(reader, "INT"); + assertNumericNull(reader, "LONG"); + assertNumericNull(reader, "FLOAT"); + assertNumericNull(reader, "DOUBLE"); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNullStringSupportsLongAndUuidTargets() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.stringColumn("u", null).stringColumn("l", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testParameterExtensionRejectedOnlyWhenColumnUsed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("future", ColumnType.LONG, new byte[]{1}), column("plain", ColumnType.LONG)); + rows.longColumn("plain", 1); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 1); + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.longColumn("future", 2)); + } + }); + } + + @Test + public void testUnknownFullTypeFlagsRejectedOnlyWhenUsed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("future", ColumnType.LONG | 0x10000), column("plain", ColumnType.LONG)); + rows.longColumn("plain", 7); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 7); + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.longColumn("future", 2)); + } + }); + } + + @Test + public void testCancelAndResetRemoveOnlyIntendedRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("l", ColumnType.LONG)); + rows.longColumn("l", 1); + buffer.nextRow(); + rows.longColumn("l", 2); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("l", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 1, 3); + buffer.reset(); + rows.longColumn("l", 4); + buffer.nextRow(); + size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 4); + } + }); + } + + @Test + public void testOwnerRollbackAfterSetterFailurePreservesCompletedRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.longColumn("l", 1); + buffer.nextRow(); + rows.longColumn("l", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.stringColumn("u", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG, QwpConstants.TYPE_UUID); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); + Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testAttachmentLifecycleAndStaleBindingGuards() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaResponse schema = known(column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + QwpSchemaBinding old = new QwpSchemaBinding(buffer, schema); + assertIllegalState(() -> new QwpSchemaBinding(buffer, schema)); + buffer.reset(); + old.longColumn("l", 1); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + buffer.clear(); + + QwpSchemaBinding current = new QwpSchemaBinding(buffer, schema); + current.longColumn("l", 7); + assertIllegalState(() -> old.longColumn("l", 8)); + assertIllegalState(() -> old.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef")); + assertIllegalState(() -> old.uuidColumn("u", 1, 2)); + buffer.nextRow(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(7, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testBindingRequiresEmptyLayout() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + buffer.getOrCreateColumn("l", QwpConstants.TYPE_LONG, true).addLong(1); + buffer.nextRow(); + buffer.reset(); + Assert.assertEquals(1, buffer.getColumnCount()); + assertIllegalState(() -> new QwpSchemaBinding(buffer, known(column("l", ColumnType.LONG)))); + } + } + + @Test + public void testBoundEncodingGuardsDoNotMutateEncoder() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer sentinel = new QwpTableBuffer("s"); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + sentinel.getOrCreateColumn("x", QwpConstants.TYPE_LONG, true).addLong(9); + sentinel.nextRow(); + encoder.encode(sentinel); + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + assertEncoderUnchanged(encoder, () -> encoder.encode(buffer)); + assertEncoderUnchanged(encoder, () -> encoder.encodeWithDeltaDict( + buffer, new GlobalSymbolDictionary(), -1, -1)); + assertEncoderUnchangedIllegalArgument(encoder, () -> encoder.encodeSchema( + buffer, binding.getTableId() + 1, binding.getMetadataVersion())); + binding.longColumn("l", 1); + assertEncoderUnchanged(encoder, () -> encoder.encodeSchema(buffer)); + buffer.nextRow(); + int derivedSize = encoder.encodeSchema(buffer); + byte[] derived = copyFrame(encoder, derivedSize); + int explicitSize = encoder.encodeSchema(buffer, binding.getTableId(), binding.getMetadataVersion()); + Assert.assertEquals(derivedSize, explicitSize); + Assert.assertArrayEquals(derived, copyFrame(encoder, explicitSize)); + + encoder.beginMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertEncoderUnchanged(encoder, () -> encoder.addTable(buffer)); + } + }); + } + + @Test + public void testEmptyBoundEncodingContracts() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + Assert.assertEquals(0, encoder.encodeSchema(buffer)); + Assert.assertEquals(0, encoder.getBuffer().getPosition()); + encoder.beginSchemaMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertEncoderUnchanged(encoder, () -> encoder.addSchemaTable( + buffer, binding.getTableId(), binding.getMetadataVersion())); + } + }); + } + + @Test + public void testUuidConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 3, fields.length); + CharSequence input = "".equals(fields[1]) ? null : fields[1]; + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + if ("".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.stringColumn("u", input)); + } else { + rows.stringColumn("u", input); + buffer.nextRow(); + } + } + } + } + } + + @Test + public void testStringAndTypedUuidEncodeSameLimbs() throws Exception { + assertMemoryLeak(() -> { + long hi = 0x0123456789abcdefL; + long lo = 0xfedcba9876543210L; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + rows.stringColumn("u", "01234567-89ab-cdef-fedc-ba9876543210"); + buffer.nextRow(); + rows.uuidColumn("u", lo, hi); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(lo, reader.longValue()); + Assert.assertEquals(hi, reader.longValue()); + Assert.assertEquals(lo, reader.longValue()); + Assert.assertEquals(hi, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + private static void assertCtorReason(int result, LineSenderSchemaException.Reason reason, String message) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + try { + new QwpSchemaBinding(buffer, result(result)); + Assert.fail("expected schema exception"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + Assert.assertFalse(e.isRetryable()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains(message)); + } + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + binding.longColumn("l", 7); + buffer.nextRow(); + assertLongFrame(encoder, encoder.encodeSchema(buffer), 7); + } + } + + private static void assertLongFrame(QwpWebSocketEncoder encoder, int size, long... values) { + Reader reader = tableReader(encoder, size, values.length, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.byteValue()); + for (long value : values) { + Assert.assertEquals(value, reader.longValue()); + } + Assert.assertEquals(size, reader.position()); + } + + private static void assertNumericValue(Reader reader, String target, String expected) { + switch (target) { + case "BYTE": + Assert.assertEquals(Integer.parseInt(expected) & 0xff, reader.byteValue()); + break; + case "SHORT": + Assert.assertEquals(Integer.parseInt(expected) & 0xffff, reader.shortValue()); + break; + case "INT": + Assert.assertEquals(Integer.parseInt(expected), reader.intValue()); + break; + case "LONG": + Assert.assertEquals(Long.parseLong(expected), reader.longValue()); + break; + case "FLOAT": + Assert.assertEquals((int) Long.parseUnsignedLong(expected.substring(2), 16), reader.intValue()); + break; + case "DOUBLE": + Assert.assertEquals(Long.parseUnsignedLong(expected.substring(2), 16), reader.longValue()); + break; + default: + Assert.fail(target); + } + } + + private static void assertNumericNull(Reader reader, String target) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(target, 1, reader.byteValue()); + } + + private static int numericColumnType(String target) { + switch (target) { + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError(target); + } + } + + private static int timestampColumnType(String target) { + switch (target) { + case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; + case "TIMESTAMP_NS": return ColumnType.TIMESTAMP_NANO; + default: throw new AssertionError(target); + } + } + + private static byte timestampWireType(String target) { + switch (target) { + case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; + case "TIMESTAMP_NS": return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: throw new AssertionError(target); + } + } + + private static byte numericWireType(String target) { + switch (target) { + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError(target); + } + } + + private static byte[] copyFrame(QwpWebSocketEncoder encoder, int size) { + byte[] copy = new byte[size]; + long address = encoder.getBuffer().getBufferPtr(); + for (int i = 0; i < size; i++) { + copy[i] = Unsafe.getUnsafe().getByte(address + i); + } + return copy; + } + + private static void assertIllegalArgument(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertIllegalState(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + // expected + } + } + + private static void assertEncoderUnchanged(QwpWebSocketEncoder encoder, Runnable action) { + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + assertIllegalState(action); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + + private static void assertEncoderUnchangedIllegalArgument(QwpWebSocketEncoder encoder, Runnable action) { + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + assertIllegalArgument(action); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + try { + action.run(); + Assert.fail("expected schema exception"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + Assert.assertFalse(e.isRetryable()); + } + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); + } + + private static QwpSchemaResponse known(byte[]... columns) { + return known(-1, columns); + } + + private static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(frame(payload.array())); + } + + private static QwpSchemaResponse result(int result) { + return decode(frame(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) result).array())); + } + + private static QwpSchemaBinding rows(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static byte[] frame(byte[] payload) { + return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload).array(); + } + + private static QwpSchemaResponse decode(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte... wireTypes) { + return tableReader(encoder, size, 1, 1, rows, wireTypes); + } + + private static Reader tableReader( + QwpWebSocketEncoder encoder, + int size, + int tableId, + long metadataVersion, + int rows, + byte... wireTypes + ) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.intValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.byteValue()); + Assert.assertEquals(1, reader.shortValue()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.intValue()); + Assert.assertEquals("t", reader.stringValue()); + if (tableId < 0) { + Assert.assertEquals(0, reader.byteValue()); + } else { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(tableId, reader.intValue()); + Assert.assertEquals(metadataVersion, reader.longValue()); + } + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(wireTypes.length, reader.varint()); + for (byte wireType : wireTypes) { + reader.stringValue(); + Assert.assertEquals(wireType, reader.byteValue()); + } + return reader; + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int byteValue() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private String ascii(int length) { + StringBuilder sink = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sink.append((char) byteValue()); + } + return sink.toString(); + } + + private int intValue() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long longValue() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private int position() { + return position; + } + + private int shortValue() { + Assert.assertTrue(position + 2 <= limit); + int value = Unsafe.getUnsafe().getShort(address + position) & 0xffff; + position += 2; + return value; + } + + private String stringValue() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) byteValue(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = byteValue(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java new file mode 100644 index 000000000..74358f1f2 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java @@ -0,0 +1,477 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/text-to-text.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 7, fields.length); + try { + String value = "".equals(fields[3]) ? null : utf16(fields[3]); + byte wireType = wireType(fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); + append(binding, fields[1], value); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType); + if (value == null) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.intValue()); + } else { + Assert.assertEquals(0, reader.varint()); + } + } else if (wireType == QwpConstants.TYPE_VARCHAR) { + byte[] expected = hex(fields[5]); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + count++; + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + } + Assert.assertEquals(90, count); + }); + } + + @Test + public void testFirstValueNullOmissionAndMutableInput() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("text", ColumnType.STRING), column("sym", ColumnType.SYMBOL)); + StringBuilder mutable = new StringBuilder("before"); + binding.symbol("text", mutable).stringColumn("text", "ignored"); + binding.stringColumn("sym", "first").symbol("sym", null); + mutable.replace(0, mutable.length(), "after"); + buffer.nextRow(); + binding.stringColumn("text", null).symbol("sym", null); + buffer.nextRow(); + buffer.nextRow(); + Assert.assertEquals("before".length(), buffer.getColumn(0).getStringDataSize()); + Assert.assertArrayEquals("before".getBytes(StandardCharsets.UTF_8), + bytes(buffer.getColumn(0).getStringDataAddress(), "before".length())); + Assert.assertEquals("first", buffer.getColumn(1).getSymbolValue(0).toString()); + int size = encoder.encodeSchema(buffer); + Assert.assertTrue(size > QwpConstants.HEADER_SIZE); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); + } + + @Test + public void testFailureRollbackAndLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaResponse schema = known(-1, + column("a", ColumnType.STRING), column("only_b", ColumnType.SYMBOL), + column("bad", ColumnType.UUID), column("c", ColumnType.VARCHAR)); + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, schema); + binding.stringColumn("a", "A"); + buffer.nextRow(); + binding.symbol("only_b", "B"); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("bad", "invalid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.symbol("c", "C"); + buffer.nextRow(); + Assert.assertEquals(2, buffer.getColumnCount()); + Assert.assertEquals("a", buffer.getColumnDefs()[0].getName()); + Assert.assertEquals("c", buffer.getColumnDefs()[1].getName()); + Assert.assertTrue(encoder.encodeSchema(buffer) > 0); + + buffer.reset(); + binding.symbol("a", "reset"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(1, reset.intValue()); + Assert.assertEquals(1, reset.longValue()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.byteValue()); + Assert.assertEquals("c", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.byteValue()); + Assert.assertEquals(0, reset.byteValue()); + Assert.assertEquals(0, reset.intValue()); + Assert.assertEquals(5, reset.intValue()); + Assert.assertArrayEquals("reset".getBytes(StandardCharsets.UTF_8), reset.bytes(5)); + Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(0, reset.intValue()); + Assert.assertEquals(resetSize, reset.position()); + buffer.clear(); + assertIllegalState(() -> binding.symbol("a", "stale")); + assertIllegalState(() -> binding.stringColumn("a", "stale")); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("long", ColumnType.LONG), column("future", ColumnType.SYMBOL, new byte[]{1}), + column("flagged", ColumnType.STRING | 0x10000)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("long", null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("future", "x")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("flagged", "x")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.symbol("missing", "x"); + Assert.assertEquals(QwpConstants.TYPE_SYMBOL, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(0, column("d", ColumnType.SYMBOL))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("d", null)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("d", null)); + } + } + + @Test + public void testUnsupportedDuplicateIsIgnoredAcrossTextSetters() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("uuid", ColumnType.UUID), column("long", ColumnType.LONG)); + binding.stringColumn("uuid", "01234567-89ab-cdef-0123-456789abcdef") + .symbol("uuid", null); + binding.longColumn("long", 7).symbol("long", null); + buffer.nextRow(); + Assert.assertTrue(encoder.encodeSchema(buffer) > 0); + } + }); + } + + @Test + public void testOwnerBackedGlobalSymbolIdsAndEncoding() throws Exception { + assertMemoryLeak(() -> { + TestWebSocketServer.WebSocketServerHandler handler = new TestWebSocketServer.WebSocketServerHandler() { }; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender senderApi = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + QwpWebSocketSender sender = (QwpWebSocketSender) senderApi; + Assert.assertEquals(0, sender.getOrAddGlobalSymbol("prefix")); + GlobalSymbolDictionary mirror = new GlobalSymbolDictionary(); + Assert.assertEquals(0, mirror.getOrAddSymbol("prefix")); + try (QwpTableBuffer buffer = new QwpTableBuffer("t", sender); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + StringBuilder mutable = new StringBuilder("mutable"); + binding.symbol("value", mutable); + mutable.replace(0, mutable.length(), "changed"); + buffer.nextRow(); + binding.stringColumn("value", "?"); + buffer.nextRow(); + binding.symbol("value", "\ud800"); + buffer.nextRow(); + binding.symbol("value", "mutable"); + buffer.nextRow(); + + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("mutable")); + Assert.assertEquals(2, sender.getOrAddGlobalSymbol("?")); + Assert.assertEquals(3, sender.getOrAddGlobalSymbol("\ud800")); + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("mutable")); + mirror.getOrAddSymbol("mutable"); + mirror.getOrAddSymbol("?"); + mirror.getOrAddSymbol("\ud800"); + Assert.assertEquals(0, buffer.getColumn(0).getSymbolDictionarySize()); + + encoder.beginSchemaMessage(1, mirror, -1, 3); + encoder.addSchemaTable(buffer, binding.getTableId(), binding.getMetadataVersion()); + int size = encoder.finishMessage(); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(4, reader.varint()); + Assert.assertArrayEquals("prefix".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("mutable".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); + Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); + skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 4); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + } + }); + } + + private static void append(QwpSchemaBinding binding, String inputType, CharSequence value) { + if ("STRING".equals(inputType)) { + binding.stringColumn("value", value); + } else if ("SYMBOL".equals(inputType)) { + binding.symbol("value", value); + } else { + throw new AssertionError("unknown input type: " + inputType); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(-1, columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type) + .putShort((short) params.length).put(params).array(); + } + + private static byte[] frame(byte[] payload) { + return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload).array(); + } + + private static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + byte[] frame = frame(payload.array()); + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTableHeader(reader, wireType, 1); + return reader; + } + + private static void skipTableHeader(Reader reader, byte wireType, int rows) { + Assert.assertEquals("t", new String(reader.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", new String(reader.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(wireType, reader.byteValue()); + } + + private static int targetType(String value) { + switch (value) { + case "STRING": + return ColumnType.STRING; + case "VARCHAR": + return ColumnType.VARCHAR; + case "SYMBOL": + return ColumnType.SYMBOL; + default: + throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "VARCHAR": + return QwpConstants.TYPE_VARCHAR; + case "SYMBOL": + return QwpConstants.TYPE_SYMBOL; + default: + throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + Assert.assertEquals(0, value.length() & 1); + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + + private static byte[] bytes(long address, int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = Unsafe.getUnsafe().getByte(address + i); + } + return value; + } + + private static String utf16(String value) { + Assert.assertEquals(0, value.length() & 3); + char[] chars = new char[value.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(value.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static void assertIllegalState(Runnable action) { + Assert.assertThrows(IllegalStateException.class, action::run); + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(reason, error.getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int byteValue() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private byte[] bytes(int length) { + Assert.assertTrue(position + length <= limit); + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = (byte) byteValue(); + } + return value; + } + + private int intValue() { + Assert.assertTrue(position + Integer.BYTES <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += Integer.BYTES; + return value; + } + + private long longValue() { + Assert.assertTrue(position + Long.BYTES <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += Long.BYTES; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + Assert.assertTrue(position + length <= limit); + position += length; + } + + private byte[] stringBytes() { + return bytes(varint()); + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = byteValue(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java new file mode 100644 index 000000000..34f72585d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java @@ -0,0 +1,337 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +public class QwpSchemaBindingTimestampTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv"; + + @Test + public void testCorpusUsesExactVarcharBytes() throws Exception { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + boolean headerSeen = false; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals( + "# case_id\tinput_kind\tunit\tvalue\tnanos\toutcome\texpected_text", line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertTrue("LONG".equals(fields[1]) || "INSTANT".equals(fields[1])); + Assert.assertTrue("VALUE".equals(fields[5]) || "INVALID".equals(fields[5])); + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + Runnable append = () -> append(binding, fields); + if ("INVALID".equals(fields[5])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + continue; + } + append.run(); + buffer.nextRow(); + Reader reader = tableReader( + encoder, encoder.encodeSchema(buffer), 1, QwpConstants.TYPE_VARCHAR); + String expectedText = "".equals(fields[6]) ? "" : fields[6]; + byte[] expected = expectedText.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(reader.limit, reader.position); + } catch (AssertionError e) { + throw new AssertionError( + "case_id=" + fields[0] + ", target=" + target + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(63, count); + } + + private static void append(QwpSchemaBinding binding, String[] fields) { + if ("INSTANT".equals(fields[1])) { + binding.timestampColumn("value", Instant.ofEpochSecond( + Long.parseLong(fields[3]), Integer.parseInt(fields[4]))); + } else if ("LONG".equals(fields[1])) { + binding.timestampColumn("value", Long.parseLong(fields[3]), ChronoUnit.valueOf(fields[2])); + } else { + throw new AssertionError("unknown input kind: " + fields[1]); + } + } + + @Test + public void testUnitsInstantNullDuplicateOmissionRollbackAndReset() { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", target), column("bad", ColumnType.UUID), column("c", target)); + binding.timestampColumn("a", Long.MIN_VALUE, ChronoUnit.MICROS) + .binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.timestampColumn("a", -1, ChronoUnit.NANOS); + buffer.nextRow(); + buffer.nextRow(); + binding.timestampColumn("a", Instant.ofEpochSecond(-1, 999_999_999)); + buffer.nextRow(); + binding.timestampColumn("a", 1, ChronoUnit.DAYS); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("bad", 1, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.timestampColumn("c", 1000, ChronoUnit.NANOS); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 5, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarchar(reader, 0x14, "", "1970-01-01T00:00:00.000Z", "1969-12-31T23:59:59.999Z"); + assertVarchar(reader, 0x0f, "1970-01-01T00:00:00.000Z"); + Assert.assertEquals(size, reader.position); + + buffer.reset(); + binding.timestampColumn("a", 0, ChronoUnit.SECONDS); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + assertVarchar(reset, -1, "1970-01-01T00:00:00.000Z"); + assertVarchar(reset, 0x01); + Assert.assertEquals(resetSize, reset.position); + } + } + } + + @Test + public void testInferenceDesignatedAndRangeErrors() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.timestampColumn("value", Long.MIN_VALUE, ChronoUnit.NANOS); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.VARCHAR)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", (Instant) null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", 1, null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("value", 1, ChronoUnit.MONTHS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.timestampColumn("value", 1, ChronoUnit.MICROS); + binding.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); + buffer.nextRow(); + } + } + + private static void assertVarchar(Reader reader, int bitmap, String... values) { + if (bitmap < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + } + int total = 0; + StringBuilder joined = new StringBuilder(); + Assert.assertEquals(0, reader.i32()); + for (String value : values) { + total += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(total, reader.i32()); + joined.append(value); + } + Assert.assertArrayEquals(joined.toString().getBytes(StandardCharsets.UTF_8), reader.bytes(total)); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte type) { + Reader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private byte[] bytes(int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return bytes; + } + + private int i32() { + Assert.assertTrue(position + 4 <= limit); + int value = Unsafe.getUnsafe().getInt(address + position); + position += 4; + return value; + } + + private long i64() { + Assert.assertTrue(position + 8 <= limit); + long value = Unsafe.getUnsafe().getLong(address + position); + position += 8; + return value; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + return new String(bytes(varint()), StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int u16() { + return u8() | (u8() << 8); + } + + private int varint() { + int result = 0; + int shift = 0; + int value; + do { + value = u8(); + result |= (value & 0x7f) << shift; + shift += 7; + } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java new file mode 100644 index 000000000..305f136be --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java @@ -0,0 +1,336 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingUuidTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/uuid-to-text.tsv"; + + @Test + public void testCorpusUsesCanonicalVarcharWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingUuidTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertEquals("VARCHAR", fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[3]))); + binding.uuidColumn("value", Long.parseLong(fields[1]), Long.parseLong(fields[2])); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1); + if ("".equals(fields[5])) { + Assert.assertEquals("", fields[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.i32()); + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals(fields[6], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(16, count); + }); + } + + @Test + public void testDuplicateNullOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType)); + binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE) + .binaryColumn("value", new byte[]{1}) + .longColumn("value", 42); + buffer.nextRow(); + binding.uuidColumn("value", 1, 0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(36, reader.i32()); + Assert.assertEquals("00000000-0000-0000-0000-000000000001", reader.ascii(36)); + Assert.assertEquals(size, reader.position()); + } + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.STRING), column("bad", ColumnType.LONG), column("c", ColumnType.VARCHAR)); + binding.uuidColumn("a", 1, 0); + buffer.nextRow(); + binding.uuidColumn("a", 2, 0); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.uuidColumn("bad", 3, 0)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.uuidColumn("c", 4, 0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarcharValues(reader, 6, "00000000-0000-0000-0000-000000000001"); + assertVarcharValues(reader, 5, "00000000-0000-0000-0000-000000000004"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.uuidColumn("a", 5, 0); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(36, reset.i32()); + Assert.assertEquals("00000000-0000-0000-0000-000000000005", reset.ascii(36)); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNativeBothMinRemainUuidBytes() { + assertNativeBothMin(response(column("value", ColumnType.UUID)), true); + assertNativeBothMin(missing(), false); + } + + private static void assertNativeBothMin(QwpSchemaResponse schema, boolean known) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, schema); + binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(known ? 1 : 0, reader.u8()); + if (known) { + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + } + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.MIN_VALUE, reader.i64()); + Assert.assertEquals(Long.MIN_VALUE, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testSymbolParameterizedAndDesignatedTargetsRemainUnsupported() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding ordinary = binding(buffer, + column("sym", ColumnType.SYMBOL), parameterizedColumn("text", ColumnType.STRING), + column("ts", ColumnType.TIMESTAMP)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> ordinary.uuidColumn("sym", Long.MIN_VALUE, Long.MIN_VALUE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> ordinary.uuidColumn("text", 1, 0)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding designated = new QwpSchemaBinding(buffer, response(0, column("ts", ColumnType.TIMESTAMP))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> designated.uuidColumn("ts", 1, 0)); + } + } + + private static void assertVarcharValues(Reader reader, int bitmap, String value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(36, reader.i32()); + Assert.assertEquals(value, reader.ascii(36)); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, response(columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] parameterizedColumn(String name, int type) { + return column(name, type, new byte[]{1}); + } + + private static byte[] column(String name, int type, byte[] parameters) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6 + parameters.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) parameters.length) + .put(parameters).array(); + } + + private static QwpSchemaResponse missing() { + ByteBuffer payload = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING); + return decode(payload.array()); + } + + private static QwpSchemaResponse response(byte[]... columns) { + return response(-1, columns); + } + + private static QwpSchemaResponse response(int designatedIndex, byte[]... columns) { + int length = 26; + for (byte[] column : columns) length += column.length; + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) payload.put(column); + return decode(payload.array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + return tableReader(encoder, size, rows, "value"); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(column, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) return ColumnType.STRING; + if ("VARCHAR".equals(value)) return ColumnType.VARCHAR; + throw new AssertionError(value); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(reason, error.getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private String ascii(int length) { return new String(bytes(length), StandardCharsets.UTF_8); } + private byte[] bytes(int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) value[i] = (byte) u8(); + return value; + } + private int i32() { int value = Unsafe.getUnsafe().getInt(address + position); position += 4; return value; } + private long i64() { long value = Unsafe.getUnsafe().getLong(address + position); position += 8; return value; } + private int position() { return position; } + private void skip(int length) { position += length; Assert.assertTrue(position <= limit); } + private String string() { return ascii(varint()); } + private int u8() { Assert.assertTrue(position < limit); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } + private int varint() { + int result = 0; + int shift = 0; + int value; + do { value = u8(); result |= (value & 0x7f) << shift; shift += 7; } while ((value & 0x80) != 0); + return result; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java new file mode 100644 index 000000000..1291802db --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java @@ -0,0 +1,323 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +public class QwpSchemaProtocolTest { + @Test + public void testAcceptsMaximumResponseColumnNameBoundaries() { + String name = repeat('\u0800', QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH); + QwpSchemaResponse response = decode(knownResponse(1, 1, 1, 0, column(name, 5, new byte[0]))); + Assert.assertEquals(name, response.getColumnName(0)); + Assert.assertEquals(QwpSchemaProtocol.MAX_NAME_UTF8_LENGTH, + name.getBytes(StandardCharsets.UTF_8).length); + } + + @Test + public void testDecodeKnownSchemaAndSkipsUnknownParams() { + byte[] frame = knownResponse(7, 42, 19, 1, + column("id", 32, new byte[]{1, 2, 3}), + column("métric🚀", 0x71234567, new byte[0])); + QwpSchemaResponse response = decode(frame); + Assert.assertEquals(7, response.getRequestId()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, response.getResult()); + Assert.assertTrue(response.hasSchema()); + Assert.assertEquals(42, response.getTableId()); + Assert.assertEquals(19, response.getMetadataVersion()); + Assert.assertEquals(1, response.getDesignatedIndex()); + Assert.assertEquals(2, response.getColumnCount()); + Assert.assertTrue(response.hasColumnExtensionParameters(0)); + Assert.assertFalse(response.hasColumnExtensionParameters(1)); + Assert.assertEquals("métric🚀", response.getColumnName(1)); + Assert.assertEquals(0x71234567, response.getColumnType(1)); + } + + @Test + public void testRejectsInvalidAndDuplicateColumnNames() { + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[0]), column("x", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[0]), column("X", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("Ä", 5, new byte[0]), column("ä", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("a/b", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("a\nb", 5, new byte[0]))); + } + + @Test + public void testAcceptsValidColumnNameCompatibilityBoundaries() { + String max = repeat('\u0800', QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH); + QwpSchemaResponse response = decode(knownResponse(1, 1, 0, -1, + column("metric name", 5, new byte[0]), + column("метрика", 5, new byte[0]), + column("!^[$", 5, new byte[0]), + column(max, 5, new byte[0]))); + Assert.assertEquals(4, response.getColumnCount()); + Assert.assertEquals(max, response.getColumnName(3)); + } + + @Test + public void testDecodeEverySchemaLessResult() { + for (int result = QwpSchemaProtocol.RESULT_MISSING; result <= QwpSchemaProtocol.RESULT_TOO_LARGE; result++) { + QwpSchemaResponse response = decode(resultResponse(11, result)); + Assert.assertEquals(result, response.getResult()); + Assert.assertFalse(response.hasSchema()); + Assert.assertEquals(0, response.getColumnCount()); + Assert.assertEquals(-1, response.getTableId()); + } + } + + @Test + public void testEncodeDescribeExactAsciiWire() { + byte[] encoded = QwpSchemaProtocol.encodeDescribe(0x0102030405060708L, "trades"); + ByteBuffer b = ByteBuffer.wrap(encoded).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, b.getInt()); + Assert.assertEquals(1, b.get() & 0xff); + Assert.assertEquals(QwpSchemaProtocol.FLAG_CONTROL, b.get()); + Assert.assertEquals(0, b.getShort()); + Assert.assertEquals(encoded.length - QwpConstants.HEADER_SIZE, b.getInt()); + Assert.assertEquals(QwpSchemaProtocol.KIND_DESCRIBE, b.get()); + Assert.assertEquals(0x0102030405060708L, b.getLong()); + Assert.assertEquals(6, b.getShort()); + byte[] name = new byte[6]; + b.get(name); + Assert.assertArrayEquals("trades".getBytes(StandardCharsets.UTF_8), name); + Assert.assertFalse(b.hasRemaining()); + } + + @Test + public void testEncodeDescribeUnicodeBoundariesAndRejectsInvalidNames() { + String maxUtf16AndUtf8 = repeat('\u0800', 127); + byte[] encoded = QwpSchemaProtocol.encodeDescribe(1, maxUtf16AndUtf8); + Assert.assertEquals(381, Short.toUnsignedInt(ByteBuffer.wrap(encoded, 21, 2).order(ByteOrder.LITTLE_ENDIAN).getShort())); + + assertEncodeRejected(0, "table"); + assertEncodeRejected(1, ""); + assertEncodeRejected(1, repeat('a', 128)); + assertEncodeRejected(1, repeat('\u0800', 126) + "🚀"); // 128 UTF-16 units + assertEncodeRejected(1, "bad\uD800name"); + assertEncodeRejected(1, "bad\uDC00name"); + assertEncodeRejected(1, "bad/name"); + } + + @Test + public void testRejectsMalformedEnvelopesAndTrailingData() { + byte[] valid = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); + assertMalformed(mutate(valid, 0, (byte) 'X')); + assertMalformed(mutate(valid, 4, (byte) 2)); + assertMalformed(mutate(valid, 5, (byte) (QwpSchemaProtocol.FLAG_CONTROL | 1))); + assertMalformed(mutate(valid, 6, (byte) 1)); + assertMalformed(mutate(valid, 12, QwpSchemaProtocol.KIND_DESCRIBE)); + assertMalformed(Arrays.copyOf(valid, valid.length - 1)); + + byte[] trailing = Arrays.copyOf(valid, valid.length + 1); + ByteBuffer.wrap(trailing).order(ByteOrder.LITTLE_ENDIAN).putInt(8, trailing.length - 12); + assertMalformed(trailing); + byte[] oversized = new byte[QwpSchemaProtocol.MAX_FRAME_SIZE + 1]; + assertMalformed(oversized); + } + + @Test + public void testPeekResponseRequestIdValidatesOnlyFixedEnvelopeAndIdentity() { + byte[] valid = resultResponse(7, QwpSchemaProtocol.RESULT_MISSING); + byte[] noResult = responseIdentity(7); + Assert.assertEquals(7, peek(noResult)); + Assert.assertEquals(7, peek(valid)); + assertMalformed(Arrays.copyOf(noResult, 20), true); + assertMalformed(mutate(valid, 0, (byte) 'X'), true); + assertMalformed(mutate(valid, 4, (byte) 2), true); + assertMalformed(mutate(valid, 5, (byte) (QwpSchemaProtocol.FLAG_CONTROL | 1)), true); + assertMalformed(mutate(valid, 6, (byte) 1), true); + assertMalformed(mutate(valid, 12, QwpSchemaProtocol.KIND_DESCRIBE), true); + assertMalformed(resultResponse(0, QwpSchemaProtocol.RESULT_MISSING), true); + + byte[] badLength = valid.clone(); + ByteBuffer.wrap(badLength).order(ByteOrder.LITTLE_ENDIAN).putInt(8, 9); + assertMalformed(badLength, true); + + // Identification deliberately stops before the result/body, while full decoding stays strict. + assertMalformed(noResult); + } + + @Test + public void testRejectsEveryTruncatedKnownResponsePrefix() { + byte[] valid = knownResponse(9, 7, 6, 1, + column("first", 5, new byte[]{1, 2, 3}), + column("секунд", 0x71234567, new byte[0])); + Assert.assertEquals(2, decode(valid).getColumnCount()); + for (int length = 0; length < valid.length; length++) { + byte[] prefix = Arrays.copyOf(valid, length); + if (length >= QwpConstants.HEADER_SIZE) { + ByteBuffer.wrap(prefix).order(ByteOrder.LITTLE_ENDIAN) + .putInt(8, length - QwpConstants.HEADER_SIZE); + } + assertMalformed(prefix); + } + } + + @Test + public void testRejectsInvalidResultsIdentitiesCountsAndIndexes() { + assertMalformed(resultResponse(0, QwpSchemaProtocol.RESULT_MISSING)); + assertMalformed(resultResponse(1, 5)); + assertMalformed(knownResponse(1, -1, 0, -1)); + assertMalformed(knownResponse(1, 1, -1, -1)); + assertMalformed(knownResponse(1, 1, 0, 0)); + assertMalformed(knownResponse(1, 1, 0, -2)); + + byte[] count = knownResponse(1, 1, 0, -1); + ByteBuffer.wrap(count).order(ByteOrder.LITTLE_ENDIAN).putShort(36, (short) 2049); + assertMalformed(count); + } + + @Test + public void testRejectsInvalidUtf8AndFieldBoundsBeforeAllocation() { + byte[] invalidUtf8 = knownResponse(1, 1, 0, -1, columnBytes(new byte[]{(byte) 0xc3, 0x28}, 5, new byte[0])); + assertMalformed(invalidUtf8); + assertMalformed(knownResponse(1, 1, 0, -1, column("", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column(repeat('a', 128), 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[1025]))); + + byte[] truncatedName = knownResponse(1, 1, 0, -1, column("abc", 5, new byte[0])); + ByteBuffer.wrap(truncatedName).order(ByteOrder.LITTLE_ENDIAN).putShort(38, (short) 381); + assertMalformed(truncatedName); + byte[] truncatedParams = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); + ByteBuffer.wrap(truncatedParams).order(ByteOrder.LITTLE_ENDIAN).putShort(45, (short) 1024); + assertMalformed(truncatedParams); + } + + private static void assertEncodeRejected(long requestId, String name) { + try { + QwpSchemaProtocol.encodeDescribe(requestId, name); + Assert.fail("expected invalid describe request"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertMalformed(byte[] frame) { + try { + decode(frame); + Assert.fail("expected malformed frame"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Malformed QWP schema response:")); + } + } + + private static void assertMalformed(byte[] frame, boolean peek) { + try { + if (peek) { + peek(frame); + } else { + decode(frame); + } + Assert.fail("expected malformed frame"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Malformed QWP schema response:")); + } + } + + private static byte[] column(String name, int type, byte[] params) { + return columnBytes(name.getBytes(StandardCharsets.UTF_8), type, params); + } + + private static byte[] columnBytes(byte[] name, int type, byte[] params) { + ByteBuffer b = ByteBuffer.allocate(2 + name.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN); + b.putShort((short) name.length).put(name).putInt(type).putShort((short) params.length).put(params); + return b.array(); + } + + private static QwpSchemaResponse decode(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static long peek(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.peekResponseRequestId(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static byte[] frame(byte[] payload) { + ByteBuffer b = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN); + b.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + return b.array(); + } + + private static byte[] knownResponse(long requestId, int tableId, long version, int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) length += column.length; + ByteBuffer b = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(tableId).putLong(version).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) b.put(column); + return frame(b.array()); + } + + private static byte[] mutate(byte[] source, int offset, byte value) { + byte[] copy = source.clone(); + copy[offset] = value; + return copy; + } + + private static String repeat(char c, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + private static byte[] resultResponse(long requestId, int result) { + ByteBuffer b = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + return frame(b.array()); + } + + private static byte[] responseIdentity(long requestId) { + ByteBuffer b = ByteBuffer.allocate(9).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId); + return frame(b.array()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 908e3bd56..788e772b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -125,6 +125,7 @@ public class TestWebSocketServer implements Closeable { // builds. Lets a test force the delta-dictionary catch-up to split across // several frames. Live-updatable via setAdvertisedMaxBatchSize(). private volatile int advertisedMaxBatchSize; + private volatile boolean advertiseSchema; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -190,6 +191,10 @@ public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedExcepti return startLatch.await(timeout, unit); } + public void setAdvertiseSchema(boolean advertiseSchema) { + this.advertiseSchema = advertiseSchema; + } + @Override public void close() { running.set(false); @@ -419,6 +424,12 @@ public void start() throws IOException { public interface WebSocketServerHandler { default void onBinaryMessage(ClientHandler client, byte[] data) { } + + default void onOpen(ClientHandler client) { + } + + default void onPong(ClientHandler client, byte[] data) { + } } /** @@ -463,6 +474,10 @@ public synchronized void sendBinary(byte[] data) throws IOException { writeFrame(WebSocketOpcode.BINARY, data, data.length); } + public synchronized void sendPing(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.PING, data, data.length); + } + public synchronized void sendClose(int code, String reason) throws IOException { byte[] reasonBytes = (reason != null && !reason.isEmpty()) ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; @@ -543,6 +558,9 @@ private void handleRead() { LOG.error("Failed to send pong", e); } break; + case WebSocketOpcode.PONG: + handler.onPong(this, payload); + break; case WebSocketOpcode.CLOSE: { int code = WebSocketCloseCode.NORMAL_CLOSURE; if (payload.length >= 2) { @@ -657,6 +675,9 @@ private boolean performHandshake() throws IOException { if (emitDurableAckHeader && !suppressDurableAckHeader) { sb.append("X-QWP-Durable-Ack: enabled\r\n"); } + if (advertiseSchema && request.toString().toLowerCase(java.util.Locale.ROOT).contains("x-qwp-request-schema: true\r\n")) { + sb.append("X-QWP-Schema: enabled\r\n"); + } String role = advertisedRole; if (role != null) { sb.append("X-QuestDB-Role: ").append(role).append("\r\n"); @@ -715,6 +736,7 @@ void start() { liveConnections.incrementAndGet(); try { + handler.onOpen(this); // SERVER_INFO is an egress-only frame: send it only on a // read-path (query) connection. An ingest write-path // connection parses every inbound frame as an ACK and diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java index 4a481e193..39b89680f 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java @@ -891,7 +891,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal128 d = new Decimal128(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -928,7 +928,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -945,7 +945,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java index 1c461525d..a27d0e7ef 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java @@ -1329,7 +1329,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal256 d = new Decimal256(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -1440,7 +1440,7 @@ public void testOfStringExponentComplexNumbers() throws NumericException { // Complex decimal with exponent int precision = Numbers.decodeLowInt(d.ofString("123.456789e-3")); - Assert.assertEquals(10, precision); + Assert.assertEquals(9, precision); Assert.assertEquals(9, d.getScale()); Assert.assertEquals("0.123456789", d.toString()); @@ -1463,7 +1463,7 @@ public void testOfStringExponentLargeButValid() throws NumericException { // Large negative exponent within bounds precision = Numbers.decodeLowInt(d.ofString("1e-20")); - Assert.assertEquals(21, precision); + Assert.assertEquals(20, precision); Assert.assertEquals(20, d.getScale()); Assert.assertEquals("0.00000000000000000001", d.toString()); } @@ -1530,7 +1530,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -1564,7 +1564,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } @@ -1581,7 +1581,7 @@ public void testOfStringExponentWithPrecisionLimit() throws NumericException { // With scale limit that matches the result scale precision = Numbers.decodeLowInt(d.ofString("1e-3", -1, 3)); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.001", d.toString()); } @@ -1612,7 +1612,7 @@ public void testOfStringExponentWithSuffix() throws NumericException { // With m suffix precision = Numbers.decodeLowInt(d.ofString("2.5e-3m")); - Assert.assertEquals(5, precision); + Assert.assertEquals(4, precision); Assert.assertEquals(4, d.getScale()); Assert.assertEquals("0.0025", d.toString()); } @@ -1832,7 +1832,7 @@ public void testOfStringPrecisionConsistency() throws NumericException { Assert.assertEquals(6, p3); int p4 = Numbers.decodeLowInt(d.ofString("00.001")); - Assert.assertEquals(4, p4); // Leading zeros don't count toward precision + Assert.assertEquals(3, p4); // Leading zeros don't count toward precision; 0.001 needs 3 fractional digits } @Test(expected = NumericException.class) @@ -1946,25 +1946,25 @@ public void testOfStringWithNegativeExponent() throws NumericException { // Basic negative exponent int precision = Numbers.decodeLowInt(d.ofString("1.234e-2")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.01234", d.toString()); // Larger negative exponent precision = Numbers.decodeLowInt(d.ofString("5e-5")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.00005", d.toString()); // Negative exponent with E (uppercase) precision = Numbers.decodeLowInt(d.ofString("7.89E-3")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.00789", d.toString()); // Very small number precision = Numbers.decodeLowInt(d.ofString("1e-10")); - Assert.assertEquals(11, precision); + Assert.assertEquals(10, precision); Assert.assertEquals(10, d.getScale()); Assert.assertEquals("0.0000000001", d.toString()); } @@ -2066,7 +2066,7 @@ public void testOfStringZeroWithDecimal() throws NumericException { Assert.assertEquals("0", d.toString()); precision = Numbers.decodeLowInt(d.ofString("0.000m")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.000", d.toString()); diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java index 769d90120..008989f2a 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java @@ -663,7 +663,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal64 d = new Decimal64(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -700,7 +700,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -717,7 +717,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java new file mode 100644 index 000000000..b9ba771c0 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java @@ -0,0 +1,115 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.std; + +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.DecimalParser; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.NumericException; +import org.junit.Assert; +import org.junit.Test; + +public class DecimalParserServerCompatibilityTest { + @Test + public void testFullScaleAndTargetScalePaddingUseServerPrecisionRules() throws NumericException { + assertParsed("0.5", 1, 1, ".5", 1, 1); + assertParsed("0.500", 3, 3, "0.5", 3, 3); + assertParsed("0.100", 3, 3, ".1", 3, 3); + + String fraction = "0." + repeat('9', 76); + assertParsed(fraction, 76, 76, fraction, 76, 76); + } + + @Test + public void testLeadingZerosAndZeroExponentUseServerPrecisionRules() throws NumericException { + assertParsed("0", 1, 0, "0000.e80", 1, 0); + assertParsed("1", 1, 0, "0.01e2", -1, -1); + assertParsed("123", 3, 0, "00.123e3", -1, -1); + assertParsed("0", 1, 0, "0e5", -1, -1); + assertParsed("0.000", 3, 3, "0e-3", 3, 3); + } + + @Test + public void testPrecisionIsAtLeastOneAndZeroFitsFullScale() throws NumericException { + assertParsed("0", 1, 0, "0.000", -1, -1); + assertParsed("0", 1, 0, "0.", -1, -1); + assertParsed("0." + repeat('0', 76), 76, 76, "0", 76, 76); + } + + @Test + public void testSpecialValuesRequireTheCompleteToken() throws NumericException { + assertNullParsed("NaN", 38, 4); + assertNullParsed("-Infinity", 76, 8); + assertRejected("NaNjunk", 38, 4); + assertRejected("Infinitygarbage", 76, 8); + } + + @Test + public void testSuffixOrderAndExtremeExponentsAreRejectedNormally() throws NumericException { + assertParsed("1.23", 3, 2, "1.2300", 38, 2); + assertRejected("1.2300M", 38, 2); + assertRejected("1.2300fm", 38, 2); + assertRejected("1.2300mf", 38, 2); + assertRejected("1e2147483647", 76, 0); + assertRejected("1e-2147483648", 76, 0); + } + + private static void assertParsed( + String expected, + int expectedPrecision, + int expectedScale, + String input, + int precision, + int scale + ) throws NumericException { + Decimal256 viaParser = new Decimal256(); + long parserMeta = DecimalParser.parse( + viaParser, input, 0, input.length(), precision, scale, false, false); + Assert.assertEquals(expected, viaParser.toString()); + Assert.assertEquals(expectedPrecision, Numbers.decodeLowInt(parserMeta)); + Assert.assertEquals(expectedScale, Numbers.decodeHighInt(parserMeta)); + + Decimal256 viaPublicDecimal = new Decimal256(); + long decimalMeta = viaPublicDecimal.ofString( + input, 0, input.length(), precision, scale, false, false); + Assert.assertEquals(expected, viaPublicDecimal.toString()); + Assert.assertEquals(expectedPrecision, Numbers.decodeLowInt(decimalMeta)); + Assert.assertEquals(expectedScale, Numbers.decodeHighInt(decimalMeta)); + } + + private static void assertNullParsed(String input, int precision, int scale) throws NumericException { + Decimal256 viaParser = new Decimal256(); + Assert.assertEquals(0, DecimalParser.parse( + viaParser, input, 0, input.length(), precision, scale, false, false)); + Assert.assertTrue(viaParser.isNull()); + + Decimal256 viaPublicDecimal = new Decimal256(); + Assert.assertEquals(0, viaPublicDecimal.ofString( + input, 0, input.length(), precision, scale, false, false)); + Assert.assertTrue(viaPublicDecimal.isNull()); + } + + private static void assertRejected(String input, int precision, int scale) { + try { + DecimalParser.parse(new Decimal256(), input, 0, input.length(), precision, scale, false, false); + Assert.fail("expected DecimalParser to reject: " + input); + } catch (NumericException expected) { + // expected + } + try { + new Decimal256().ofString(input, 0, input.length(), precision, scale, false, false); + Assert.fail("expected Decimal256 to reject: " + input); + } catch (NumericException expected) { + // expected + } + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + for (int i = 0; i < count; i++) chars[i] = value; + return new String(chars); + } +} diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv new file mode 100644 index 000000000..21a5afe70 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv @@ -0,0 +1,21 @@ +# case_id input_kind input_data_hex expected_wire_type expected_wire_hex expected_sql_hex outcome +# BINARY input is raw bytes; STRING input is UTF-16BE code units. Empty is a present zero-length value. +# is an explicit bitmap null. Hex is lowercase and separator-free. +binary_empty BINARY BINARY VALID +binary_zero BINARY 00 BINARY 00 00 VALID +binary_opaque BINARY 00ff807f010203 BINARY 00ff807f010203 00ff807f010203 VALID +binary_all_256 BINARY 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff BINARY 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff VALID +string_null STRING BINARY VALID +string_empty STRING BINARY VALID +string_ascii STRING 00510075006500730074 BINARY 5175657374 5175657374 VALID +string_embedded_nul STRING 006100000062 BINARY 610062 610062 VALID +string_bmp_two_byte STRING 00e9 BINARY c3a9 c3a9 VALID +string_bmp_three_byte STRING 6c49 BINARY e6b189 e6b189 VALID +string_supplementary STRING d83dde00 BINARY f09f9880 f09f9880 VALID +string_lone_high STRING d800 BINARY 3f 3f VALID +string_high_ascii STRING d8000061 BINARY 3f61 3f61 VALID +string_high_high_low STRING d800d83dde00 BINARY 3ff09f9880 3ff09f9880 VALID +string_lone_low STRING dc00 BINARY 3f 3f VALID +string_low_high_ascii STRING dc00d8000061 BINARY 3f3f61 3f3f61 VALID +string_literal_question STRING 003f BINARY 3f 3f VALID +string_controls STRING 00610009000d000a0062 BINARY 61090d0a62 61090d0a62 VALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv new file mode 100644 index 000000000..8bcc6a231 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv @@ -0,0 +1,19 @@ +# case_id input_boolean target_type expected_wire_type expected_wire_value expected_sql_value +bool_false_boolean FALSE BOOLEAN BOOLEAN 0 false +bool_true_boolean TRUE BOOLEAN BOOLEAN 1 true +bool_false_byte FALSE BYTE BYTE 0 0 +bool_true_byte TRUE BYTE BYTE 1 1 +bool_false_short FALSE SHORT SHORT 0 0 +bool_true_short TRUE SHORT SHORT 1 1 +bool_false_int FALSE INT INT 0 0 +bool_true_int TRUE INT INT 1 1 +bool_false_long FALSE LONG LONG 0 0 +bool_true_long TRUE LONG LONG 1 1 +bool_false_float FALSE FLOAT FLOAT 0 0 +bool_true_float TRUE FLOAT FLOAT 1 1 +bool_false_double FALSE DOUBLE DOUBLE 0 0 +bool_true_double TRUE DOUBLE DOUBLE 1 1 +bool_false_string FALSE STRING VARCHAR 66616c7365 00660061006c00730065 +bool_true_string TRUE STRING VARCHAR 74727565 0074007200750065 +bool_false_varchar FALSE VARCHAR VARCHAR 66616c7365 00660061006c00730065 +bool_true_varchar TRUE VARCHAR VARCHAR 74727565 0074007200750065 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv new file mode 100644 index 000000000..cfb464ac3 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv @@ -0,0 +1,24 @@ +# case_id input_kind utf16_hex expected_char expected_null +char_nul CHAR 0000 0 true +char_ascii CHAR 0041 65 false +char_two_byte CHAR 00e9 233 false +char_three_byte CHAR 20ac 8364 false +char_before_surrogate CHAR d7ff 55295 false +char_high_surrogate CHAR d83d 55357 false +char_low_surrogate CHAR de00 56832 false +char_after_surrogate CHAR e000 57344 false +char_max CHAR ffff 65535 false +string_null STRING 0 true +string_empty STRING 0 true +string_nul STRING 0000 0 true +string_ascii STRING 0041 65 false +string_two_byte STRING 00e9 233 false +string_three_byte STRING 20ac 8364 false +string_before_surrogate STRING d7ff 55295 false +string_after_surrogate STRING e000 57344 false +string_max STRING ffff 65535 false +string_supplementary_first STRING d83dde00 0 true +string_lone_high STRING d83d 63 false +string_lone_low STRING de00 63 false +string_high_then_ascii STRING d83d0041 63 false +string_ascii_then_trailing STRING 004200e9d83dde00 66 false diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv new file mode 100644 index 000000000..f6468b645 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv @@ -0,0 +1,61 @@ +# case_id source_type input_bits_hex target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +float_to_decimal8 FLOAT 41180000 DECIMAL8 2 1 VALUE 000000000000005f - - - 9.5 +double_to_decimal8 DOUBLE 3fb999999999999a DECIMAL8 2 1 VALUE 0000000000000001 - - - 0.1 +float_to_decimal16 FLOAT c1480000 DECIMAL16 4 2 VALUE fffffffffffffb1e - - - -12.50 +double_to_decimal16 DOUBLE 4058ff5c28f5c28f DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +float_to_decimal32 FLOAT 4640e480 DECIMAL32 9 3 VALUE 0000000000bc5f25 - - - 12345.125 +double_to_decimal32 DOUBLE c08f3ffdf3b645a2 DECIMAL32 9 3 VALUE fffffffffff0bdc1 - - - -999.999 +float_to_decimal64 FLOAT 3dcccccd DECIMAL64 18 17 VALUE 002386f278a2bc9c - - - 0.10000000149011612 +double_to_decimal64 DOUBLE 3fb999999999999a DECIMAL64 18 17 VALUE 002386f26fc10000 - - - 0.10000000000000000 +float_to_decimal128 FLOAT 4b800000 DECIMAL128 24 5 VALUE 00000186a0000000 0000000000000000 - - 16777216.00000 +double_to_decimal128 DOUBLE 44b52d02c7e14af6 DECIMAL128 24 0 VALUE 02c7e14af6800000 000000000000152d - - 100000000000000000000000 +float_to_decimal256 FLOAT 7f7fffff DECIMAL256 76 0 VALUE 35211f7ca1800000 ffffff000000000a 0000000000000000 0000000000000000 340282346638528860000000000000000000000 +double_to_decimal256 DOUBLE 3027288e1271f513 DECIMAL256 76 76 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +decimal8_positive_limit DOUBLE 4023cccccccccccd DECIMAL8 2 1 VALUE 0000000000000063 - - - 9.9 +decimal8_negative_limit DOUBLE c023cccccccccccd DECIMAL8 2 1 VALUE ffffffffffffff9d - - - -9.9 +decimal8_positive_over FLOAT 41200000 DECIMAL8 2 1 INVALID - - - - - +decimal8_negative_over DOUBLE c024000000000000 DECIMAL8 2 1 INVALID - - - - - +decimal16_positive_limit DOUBLE 4058ff5c28f5c28f DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +decimal16_negative_fit FLOAT c2c70000 DECIMAL16 4 2 VALUE ffffffffffffd922 - - - -99.50 +decimal16_positive_over DOUBLE 4059000000000000 DECIMAL16 4 2 INVALID - - - - - +decimal16_negative_over FLOAT c2c80000 DECIMAL16 4 2 INVALID - - - - - +decimal32_positive_limit DOUBLE 412e847fff7ced91 DECIMAL32 9 3 VALUE 000000003b9ac9ff - - - 999999.999 +decimal32_positive_fit FLOAT 497423f8 DECIMAL32 9 3 VALUE 000000003b9ac80c - - - 999999.500 +decimal32_positive_over DOUBLE 412e848000000000 DECIMAL32 9 3 INVALID - - - - - +decimal32_negative_over FLOAT c9742400 DECIMAL32 9 3 INVALID - - - - - +decimal64_positive_fit DOUBLE 4376345785d89fff DECIMAL64 18 1 VALUE 0de0b6b3a763ff38 - - - 99999999999999980.0 +decimal64_positive_over DOUBLE 4376345785d8a000 DECIMAL64 18 1 INVALID - - - - - +decimal128_positive_fit DOUBLE 43abc16d674ec7ff DECIMAL128 38 20 VALUE efa958854dc00000 4b3b4ca85a86c25b - - 999999999999999900.00000000000000000000 +decimal128_positive_over DOUBLE 43abc16d674ec800 DECIMAL128 38 20 INVALID - - - - - +decimal256_positive_fit DOUBLE 43e158e460913cff DECIMAL256 76 57 VALUE e000000000000000 c8b1219848c9ee4d 6916687cb4209f75 161bcca711991476 9999999999999998000.000000000000000000000000000000000000000000000000000000000 +decimal256_positive_over DOUBLE 43e158e460913d00 DECIMAL256 76 57 INVALID - - - - - +double_no_rounding DOUBLE 3ff4000000000000 DECIMAL8 2 1 INVALID - - - - - +float_no_rounding FLOAT 3fa00000 DECIMAL8 2 1 INVALID - - - - - +widened_float_point_one_scale_reject FLOAT 3dcccccd DECIMAL8 2 1 INVALID - - - - - +double_point_one_scale_accept DOUBLE 3fb999999999999a DECIMAL8 2 1 VALUE 0000000000000001 - - - 0.1 +float_positive_zero FLOAT 00000000 DECIMAL16 4 2 VALUE 0000000000000000 - - - 0.00 +float_negative_zero FLOAT 80000000 DECIMAL32 9 3 VALUE 0000000000000000 - - - 0.000 +double_positive_zero DOUBLE 0000000000000000 DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +double_negative_zero DOUBLE 8000000000000000 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +float_nan_payload FLOAT 7fc00001 DECIMAL8 2 1 NULL - - - - NULL +float_positive_infinity FLOAT 7f800000 DECIMAL16 4 2 NULL - - - - NULL +float_negative_infinity FLOAT ff800000 DECIMAL32 9 3 NULL - - - - NULL +double_nan_payload DOUBLE 7ff8000000000042 DECIMAL64 18 8 NULL - - - - NULL +double_positive_infinity DOUBLE 7ff0000000000000 DECIMAL128 38 20 NULL - - - - NULL +double_negative_infinity DOUBLE fff0000000000000 DECIMAL256 76 57 NULL - - - - NULL +float_min_value_widened_scale_60 FLOAT 00000001 DECIMAL256 60 60 VALUE 0004fa793930bcd1 0000000000000000 0000000000000000 0000000000000000 0.000000000000000000000000000000000000000000001401298464324817 +float_min_value_widened_scale_59_reject FLOAT 00000001 DECIMAL256 59 59 INVALID - - - - - +float_max_value FLOAT 7f7fffff DECIMAL256 76 0 VALUE 35211f7ca1800000 ffffff000000000a 0000000000000000 0000000000000000 340282346638528860000000000000000000000 +double_one_e_minus_76 DOUBLE 3027288e1271f513 DECIMAL256 76 76 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +double_one_e_minus_77_reject DOUBLE 2ff286d80ec190dc DECIMAL256 76 76 INVALID - - - - - +double_min_normal_reject DOUBLE 0010000000000000 DECIMAL256 76 76 INVALID - - - - - +double_min_value_reject DOUBLE 0000000000000001 DECIMAL256 76 76 INVALID - - - - - +double_max_value_reject DOUBLE 7fefffffffffffff DECIMAL256 76 0 INVALID - - - - - +float_below_two_pow_24 FLOAT 4b7fffff DECIMAL64 18 0 VALUE 0000000000ffffff - - - 16777215 +float_two_pow_24 FLOAT 4b800000 DECIMAL64 18 0 VALUE 0000000001000000 - - - 16777216 +float_above_two_pow_24 FLOAT 4b800001 DECIMAL64 18 0 VALUE 0000000001000002 - - - 16777218 +double_below_two_pow_53 DOUBLE 433fffffffffffff DECIMAL64 18 0 VALUE 001fffffffffffff - - - 9007199254740991 +double_two_pow_53 DOUBLE 4340000000000000 DECIMAL64 18 0 VALUE 0020000000000000 - - - 9007199254740992 +double_above_two_pow_53 DOUBLE 4340000000000001 DECIMAL64 18 0 VALUE 0020000000000002 - - - 9007199254740994 +double_repeating_shortest DOUBLE 3fd5555555555555 DECIMAL64 18 17 VALUE 00766c7d74835552 - - - 0.33333333333333330 +double_large_ryu_boundary DOUBLE 44b52d02c7e14af6 DECIMAL128 24 0 VALUE 02c7e14af6800000 000000000000152d - - 100000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv new file mode 100644 index 000000000..6c5d59e36 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv @@ -0,0 +1,353 @@ +# Seven fields: case_id, input_type, input_raw_bits, target_type, expected_wire_type, expected_wire_bits, expected_sql_value +# Raw bits are lowercase fixed-width hexadecimal. is bitmap null/no value bytes; paired fields are local rejection. +# BITS: means stored floating raw bits. Present INT/LONG minimum uses numeric wire bits and SQL NULL. +float_byte_pos_zero FLOAT 00000000 BYTE BYTE 00 0 +float_byte_neg_zero FLOAT 80000000 BYTE BYTE 00 0 +float_byte_pos_one FLOAT 3f800000 BYTE BYTE 01 1 +float_byte_neg_one FLOAT bf800000 BYTE BYTE ff -1 +float_byte_fraction FLOAT 3fc00000 BYTE BYTE +float_byte_pos_inf FLOAT 7f800000 BYTE BYTE +float_byte_neg_inf FLOAT ff800000 BYTE BYTE +float_byte_nan_payload FLOAT 7fc00001 BYTE BYTE 0 +float_byte_nan_negative FLOAT ffc12345 BYTE BYTE 0 +float_byte_min_subnormal FLOAT 00000001 BYTE BYTE +float_byte_max_finite FLOAT 7f7fffff BYTE BYTE +float_byte_byte_neg_128 FLOAT c3000000 BYTE BYTE 80 -128 +float_byte_byte_pos_127 FLOAT 42fe0000 BYTE BYTE 7f 127 +float_byte_byte_neg_129 FLOAT c3010000 BYTE BYTE +float_byte_byte_pos_128 FLOAT 43000000 BYTE BYTE +float_byte_short_neg_32768 FLOAT c7000000 BYTE BYTE +float_byte_short_pos_32767 FLOAT 46fffe00 BYTE BYTE +float_byte_short_neg_32769 FLOAT c7000100 BYTE BYTE +float_byte_short_pos_32768 FLOAT 47000000 BYTE BYTE +float_byte_int_neg_2147483648 FLOAT cf000000 BYTE BYTE +float_byte_int_pos_2147483647 FLOAT 4f000000 BYTE BYTE +float_byte_int_neg_2147483649 FLOAT cf000000 BYTE BYTE +float_byte_int_pos_2147483648 FLOAT 4f000000 BYTE BYTE +float_byte_long_neg_9223372036854776000 FLOAT df000000 BYTE BYTE +float_byte_long_pos_9223372036854776000 FLOAT 5f000000 BYTE BYTE +float_short_pos_zero FLOAT 00000000 SHORT SHORT 0000 0 +float_short_neg_zero FLOAT 80000000 SHORT SHORT 0000 0 +float_short_pos_one FLOAT 3f800000 SHORT SHORT 0001 1 +float_short_neg_one FLOAT bf800000 SHORT SHORT ffff -1 +float_short_fraction FLOAT 3fc00000 SHORT SHORT +float_short_pos_inf FLOAT 7f800000 SHORT SHORT +float_short_neg_inf FLOAT ff800000 SHORT SHORT +float_short_nan_payload FLOAT 7fc00001 SHORT SHORT 0 +float_short_nan_negative FLOAT ffc12345 SHORT SHORT 0 +float_short_min_subnormal FLOAT 00000001 SHORT SHORT +float_short_max_finite FLOAT 7f7fffff SHORT SHORT +float_short_byte_neg_128 FLOAT c3000000 SHORT SHORT ff80 -128 +float_short_byte_pos_127 FLOAT 42fe0000 SHORT SHORT 007f 127 +float_short_byte_neg_129 FLOAT c3010000 SHORT SHORT ff7f -129 +float_short_byte_pos_128 FLOAT 43000000 SHORT SHORT 0080 128 +float_short_short_neg_32768 FLOAT c7000000 SHORT SHORT 8000 -32768 +float_short_short_pos_32767 FLOAT 46fffe00 SHORT SHORT 7fff 32767 +float_short_short_neg_32769 FLOAT c7000100 SHORT SHORT +float_short_short_pos_32768 FLOAT 47000000 SHORT SHORT +float_short_int_neg_2147483648 FLOAT cf000000 SHORT SHORT +float_short_int_pos_2147483647 FLOAT 4f000000 SHORT SHORT +float_short_int_neg_2147483649 FLOAT cf000000 SHORT SHORT +float_short_int_pos_2147483648 FLOAT 4f000000 SHORT SHORT +float_short_long_neg_9223372036854776000 FLOAT df000000 SHORT SHORT +float_short_long_pos_9223372036854776000 FLOAT 5f000000 SHORT SHORT +float_int_pos_zero FLOAT 00000000 INT INT 00000000 0 +float_int_neg_zero FLOAT 80000000 INT INT 00000000 0 +float_int_pos_one FLOAT 3f800000 INT INT 00000001 1 +float_int_neg_one FLOAT bf800000 INT INT ffffffff -1 +float_int_fraction FLOAT 3fc00000 INT INT +float_int_pos_inf FLOAT 7f800000 INT INT +float_int_neg_inf FLOAT ff800000 INT INT +float_int_nan_payload FLOAT 7fc00001 INT INT NULL +float_int_nan_negative FLOAT ffc12345 INT INT NULL +float_int_min_subnormal FLOAT 00000001 INT INT +float_int_max_finite FLOAT 7f7fffff INT INT +float_int_byte_neg_128 FLOAT c3000000 INT INT ffffff80 -128 +float_int_byte_pos_127 FLOAT 42fe0000 INT INT 0000007f 127 +float_int_byte_neg_129 FLOAT c3010000 INT INT ffffff7f -129 +float_int_byte_pos_128 FLOAT 43000000 INT INT 00000080 128 +float_int_short_neg_32768 FLOAT c7000000 INT INT ffff8000 -32768 +float_int_short_pos_32767 FLOAT 46fffe00 INT INT 00007fff 32767 +float_int_short_neg_32769 FLOAT c7000100 INT INT ffff7fff -32769 +float_int_short_pos_32768 FLOAT 47000000 INT INT 00008000 32768 +float_int_int_neg_2147483648 FLOAT cf000000 INT INT 80000000 NULL +float_int_int_pos_2147483647 FLOAT 4f000000 INT INT +float_int_int_neg_2147483649 FLOAT cf000000 INT INT 80000000 NULL +float_int_int_pos_2147483648 FLOAT 4f000000 INT INT +float_int_long_neg_9223372036854776000 FLOAT df000000 INT INT +float_int_long_pos_9223372036854776000 FLOAT 5f000000 INT INT +float_long_pos_zero FLOAT 00000000 LONG LONG 0000000000000000 0 +float_long_neg_zero FLOAT 80000000 LONG LONG 0000000000000000 0 +float_long_pos_one FLOAT 3f800000 LONG LONG 0000000000000001 1 +float_long_neg_one FLOAT bf800000 LONG LONG ffffffffffffffff -1 +float_long_fraction FLOAT 3fc00000 LONG LONG +float_long_pos_inf FLOAT 7f800000 LONG LONG +float_long_neg_inf FLOAT ff800000 LONG LONG +float_long_nan_payload FLOAT 7fc00001 LONG LONG NULL +float_long_nan_negative FLOAT ffc12345 LONG LONG NULL +float_long_min_subnormal FLOAT 00000001 LONG LONG +float_long_max_finite FLOAT 7f7fffff LONG LONG +float_long_byte_neg_128 FLOAT c3000000 LONG LONG ffffffffffffff80 -128 +float_long_byte_pos_127 FLOAT 42fe0000 LONG LONG 000000000000007f 127 +float_long_byte_neg_129 FLOAT c3010000 LONG LONG ffffffffffffff7f -129 +float_long_byte_pos_128 FLOAT 43000000 LONG LONG 0000000000000080 128 +float_long_short_neg_32768 FLOAT c7000000 LONG LONG ffffffffffff8000 -32768 +float_long_short_pos_32767 FLOAT 46fffe00 LONG LONG 0000000000007fff 32767 +float_long_short_neg_32769 FLOAT c7000100 LONG LONG ffffffffffff7fff -32769 +float_long_short_pos_32768 FLOAT 47000000 LONG LONG 0000000000008000 32768 +float_long_int_neg_2147483648 FLOAT cf000000 LONG LONG ffffffff80000000 -2147483648 +float_long_int_pos_2147483647 FLOAT 4f000000 LONG LONG 0000000080000000 2147483648 +float_long_int_neg_2147483649 FLOAT cf000000 LONG LONG ffffffff80000000 -2147483648 +float_long_int_pos_2147483648 FLOAT 4f000000 LONG LONG 0000000080000000 2147483648 +float_long_long_neg_9223372036854776000 FLOAT df000000 LONG LONG 8000000000000000 NULL +float_long_long_pos_9223372036854776000 FLOAT 5f000000 LONG LONG +float_float_pos_zero FLOAT 00000000 FLOAT FLOAT 00000000 BITS:00000000 +float_float_neg_zero FLOAT 80000000 FLOAT FLOAT 80000000 BITS:80000000 +float_float_pos_one FLOAT 3f800000 FLOAT FLOAT 3f800000 BITS:3f800000 +float_float_neg_one FLOAT bf800000 FLOAT FLOAT bf800000 BITS:bf800000 +float_float_fraction FLOAT 3fc00000 FLOAT FLOAT 3fc00000 BITS:3fc00000 +float_float_pos_inf FLOAT 7f800000 FLOAT FLOAT 7f800000 BITS:7f800000 +float_float_neg_inf FLOAT ff800000 FLOAT FLOAT ff800000 BITS:ff800000 +float_float_nan_payload FLOAT 7fc00001 FLOAT FLOAT NULL +float_float_nan_negative FLOAT ffc12345 FLOAT FLOAT NULL +float_float_min_subnormal FLOAT 00000001 FLOAT FLOAT 00000001 BITS:00000001 +float_float_max_finite FLOAT 7f7fffff FLOAT FLOAT 7f7fffff BITS:7f7fffff +float_float_byte_neg_128 FLOAT c3000000 FLOAT FLOAT c3000000 BITS:c3000000 +float_float_byte_pos_127 FLOAT 42fe0000 FLOAT FLOAT 42fe0000 BITS:42fe0000 +float_float_byte_neg_129 FLOAT c3010000 FLOAT FLOAT c3010000 BITS:c3010000 +float_float_byte_pos_128 FLOAT 43000000 FLOAT FLOAT 43000000 BITS:43000000 +float_float_short_neg_32768 FLOAT c7000000 FLOAT FLOAT c7000000 BITS:c7000000 +float_float_short_pos_32767 FLOAT 46fffe00 FLOAT FLOAT 46fffe00 BITS:46fffe00 +float_float_short_neg_32769 FLOAT c7000100 FLOAT FLOAT c7000100 BITS:c7000100 +float_float_short_pos_32768 FLOAT 47000000 FLOAT FLOAT 47000000 BITS:47000000 +float_float_int_neg_2147483648 FLOAT cf000000 FLOAT FLOAT cf000000 BITS:cf000000 +float_float_int_pos_2147483647 FLOAT 4f000000 FLOAT FLOAT 4f000000 BITS:4f000000 +float_float_int_neg_2147483649 FLOAT cf000000 FLOAT FLOAT cf000000 BITS:cf000000 +float_float_int_pos_2147483648 FLOAT 4f000000 FLOAT FLOAT 4f000000 BITS:4f000000 +float_float_long_neg_9223372036854776000 FLOAT df000000 FLOAT FLOAT df000000 BITS:df000000 +float_float_long_pos_9223372036854776000 FLOAT 5f000000 FLOAT FLOAT 5f000000 BITS:5f000000 +float_double_pos_zero FLOAT 00000000 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 +float_double_neg_zero FLOAT 80000000 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 +float_double_pos_one FLOAT 3f800000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 +float_double_neg_one FLOAT bf800000 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 +float_double_fraction FLOAT 3fc00000 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 +float_double_pos_inf FLOAT 7f800000 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 +float_double_neg_inf FLOAT ff800000 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 +float_double_nan_payload FLOAT 7fc00001 DOUBLE DOUBLE NULL +float_double_nan_negative FLOAT ffc12345 DOUBLE DOUBLE NULL +float_double_min_subnormal FLOAT 00000001 DOUBLE DOUBLE 36a0000000000000 BITS:36a0000000000000 +float_double_max_finite FLOAT 7f7fffff DOUBLE DOUBLE 47efffffe0000000 BITS:47efffffe0000000 +float_double_byte_neg_128 FLOAT c3000000 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 +float_double_byte_pos_127 FLOAT 42fe0000 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 +float_double_byte_neg_129 FLOAT c3010000 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 +float_double_byte_pos_128 FLOAT 43000000 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 +float_double_short_neg_32768 FLOAT c7000000 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 +float_double_short_pos_32767 FLOAT 46fffe00 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 +float_double_short_neg_32769 FLOAT c7000100 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 +float_double_short_pos_32768 FLOAT 47000000 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 +float_double_int_neg_2147483648 FLOAT cf000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +float_double_int_pos_2147483647 FLOAT 4f000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +float_double_int_neg_2147483649 FLOAT cf000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +float_double_int_pos_2147483648 FLOAT 4f000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +float_double_long_neg_9223372036854776000 FLOAT df000000 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 +float_double_long_pos_9223372036854776000 FLOAT 5f000000 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 +double_byte_pos_zero DOUBLE 0000000000000000 BYTE BYTE 00 0 +double_byte_neg_zero DOUBLE 8000000000000000 BYTE BYTE 00 0 +double_byte_pos_one DOUBLE 3ff0000000000000 BYTE BYTE 01 1 +double_byte_neg_one DOUBLE bff0000000000000 BYTE BYTE ff -1 +double_byte_fraction DOUBLE 3ff8000000000000 BYTE BYTE +double_byte_pos_inf DOUBLE 7ff0000000000000 BYTE BYTE +double_byte_neg_inf DOUBLE fff0000000000000 BYTE BYTE +double_byte_nan_payload DOUBLE 7ff8000000000001 BYTE BYTE 0 +double_byte_nan_negative DOUBLE fff8000000000042 BYTE BYTE 0 +double_byte_min_subnormal DOUBLE 0000000000000001 BYTE BYTE +double_byte_max_finite DOUBLE 7fefffffffffffff BYTE BYTE +double_byte_byte_neg_128 DOUBLE c060000000000000 BYTE BYTE 80 -128 +double_byte_byte_pos_127 DOUBLE 405fc00000000000 BYTE BYTE 7f 127 +double_byte_byte_neg_129 DOUBLE c060200000000000 BYTE BYTE +double_byte_byte_pos_128 DOUBLE 4060000000000000 BYTE BYTE +double_byte_short_neg_32768 DOUBLE c0e0000000000000 BYTE BYTE +double_byte_short_pos_32767 DOUBLE 40dfffc000000000 BYTE BYTE +double_byte_short_neg_32769 DOUBLE c0e0002000000000 BYTE BYTE +double_byte_short_pos_32768 DOUBLE 40e0000000000000 BYTE BYTE +double_byte_int_neg_2147483648 DOUBLE c1e0000000000000 BYTE BYTE +double_byte_int_pos_2147483647 DOUBLE 41dfffffffc00000 BYTE BYTE +double_byte_int_neg_2147483649 DOUBLE c1e0000000200000 BYTE BYTE +double_byte_int_pos_2147483648 DOUBLE 41e0000000000000 BYTE BYTE +double_byte_long_neg_9223372036854776000 DOUBLE c3e0000000000000 BYTE BYTE +double_byte_long_pos_9223372036854776000 DOUBLE 43e0000000000000 BYTE BYTE +double_short_pos_zero DOUBLE 0000000000000000 SHORT SHORT 0000 0 +double_short_neg_zero DOUBLE 8000000000000000 SHORT SHORT 0000 0 +double_short_pos_one DOUBLE 3ff0000000000000 SHORT SHORT 0001 1 +double_short_neg_one DOUBLE bff0000000000000 SHORT SHORT ffff -1 +double_short_fraction DOUBLE 3ff8000000000000 SHORT SHORT +double_short_pos_inf DOUBLE 7ff0000000000000 SHORT SHORT +double_short_neg_inf DOUBLE fff0000000000000 SHORT SHORT +double_short_nan_payload DOUBLE 7ff8000000000001 SHORT SHORT 0 +double_short_nan_negative DOUBLE fff8000000000042 SHORT SHORT 0 +double_short_min_subnormal DOUBLE 0000000000000001 SHORT SHORT +double_short_max_finite DOUBLE 7fefffffffffffff SHORT SHORT +double_short_byte_neg_128 DOUBLE c060000000000000 SHORT SHORT ff80 -128 +double_short_byte_pos_127 DOUBLE 405fc00000000000 SHORT SHORT 007f 127 +double_short_byte_neg_129 DOUBLE c060200000000000 SHORT SHORT ff7f -129 +double_short_byte_pos_128 DOUBLE 4060000000000000 SHORT SHORT 0080 128 +double_short_short_neg_32768 DOUBLE c0e0000000000000 SHORT SHORT 8000 -32768 +double_short_short_pos_32767 DOUBLE 40dfffc000000000 SHORT SHORT 7fff 32767 +double_short_short_neg_32769 DOUBLE c0e0002000000000 SHORT SHORT +double_short_short_pos_32768 DOUBLE 40e0000000000000 SHORT SHORT +double_short_int_neg_2147483648 DOUBLE c1e0000000000000 SHORT SHORT +double_short_int_pos_2147483647 DOUBLE 41dfffffffc00000 SHORT SHORT +double_short_int_neg_2147483649 DOUBLE c1e0000000200000 SHORT SHORT +double_short_int_pos_2147483648 DOUBLE 41e0000000000000 SHORT SHORT +double_short_long_neg_9223372036854776000 DOUBLE c3e0000000000000 SHORT SHORT +double_short_long_pos_9223372036854776000 DOUBLE 43e0000000000000 SHORT SHORT +double_int_pos_zero DOUBLE 0000000000000000 INT INT 00000000 0 +double_int_neg_zero DOUBLE 8000000000000000 INT INT 00000000 0 +double_int_pos_one DOUBLE 3ff0000000000000 INT INT 00000001 1 +double_int_neg_one DOUBLE bff0000000000000 INT INT ffffffff -1 +double_int_fraction DOUBLE 3ff8000000000000 INT INT +double_int_pos_inf DOUBLE 7ff0000000000000 INT INT +double_int_neg_inf DOUBLE fff0000000000000 INT INT +double_int_nan_payload DOUBLE 7ff8000000000001 INT INT NULL +double_int_nan_negative DOUBLE fff8000000000042 INT INT NULL +double_int_min_subnormal DOUBLE 0000000000000001 INT INT +double_int_max_finite DOUBLE 7fefffffffffffff INT INT +double_int_byte_neg_128 DOUBLE c060000000000000 INT INT ffffff80 -128 +double_int_byte_pos_127 DOUBLE 405fc00000000000 INT INT 0000007f 127 +double_int_byte_neg_129 DOUBLE c060200000000000 INT INT ffffff7f -129 +double_int_byte_pos_128 DOUBLE 4060000000000000 INT INT 00000080 128 +double_int_short_neg_32768 DOUBLE c0e0000000000000 INT INT ffff8000 -32768 +double_int_short_pos_32767 DOUBLE 40dfffc000000000 INT INT 00007fff 32767 +double_int_short_neg_32769 DOUBLE c0e0002000000000 INT INT ffff7fff -32769 +double_int_short_pos_32768 DOUBLE 40e0000000000000 INT INT 00008000 32768 +double_int_int_neg_2147483648 DOUBLE c1e0000000000000 INT INT 80000000 NULL +double_int_int_pos_2147483647 DOUBLE 41dfffffffc00000 INT INT 7fffffff 2147483647 +double_int_int_neg_2147483649 DOUBLE c1e0000000200000 INT INT +double_int_int_pos_2147483648 DOUBLE 41e0000000000000 INT INT +double_int_long_neg_9223372036854776000 DOUBLE c3e0000000000000 INT INT +double_int_long_pos_9223372036854776000 DOUBLE 43e0000000000000 INT INT +double_long_pos_zero DOUBLE 0000000000000000 LONG LONG 0000000000000000 0 +double_long_neg_zero DOUBLE 8000000000000000 LONG LONG 0000000000000000 0 +double_long_pos_one DOUBLE 3ff0000000000000 LONG LONG 0000000000000001 1 +double_long_neg_one DOUBLE bff0000000000000 LONG LONG ffffffffffffffff -1 +double_long_fraction DOUBLE 3ff8000000000000 LONG LONG +double_long_pos_inf DOUBLE 7ff0000000000000 LONG LONG +double_long_neg_inf DOUBLE fff0000000000000 LONG LONG +double_long_nan_payload DOUBLE 7ff8000000000001 LONG LONG NULL +double_long_nan_negative DOUBLE fff8000000000042 LONG LONG NULL +double_long_min_subnormal DOUBLE 0000000000000001 LONG LONG +double_long_max_finite DOUBLE 7fefffffffffffff LONG LONG +double_long_byte_neg_128 DOUBLE c060000000000000 LONG LONG ffffffffffffff80 -128 +double_long_byte_pos_127 DOUBLE 405fc00000000000 LONG LONG 000000000000007f 127 +double_long_byte_neg_129 DOUBLE c060200000000000 LONG LONG ffffffffffffff7f -129 +double_long_byte_pos_128 DOUBLE 4060000000000000 LONG LONG 0000000000000080 128 +double_long_short_neg_32768 DOUBLE c0e0000000000000 LONG LONG ffffffffffff8000 -32768 +double_long_short_pos_32767 DOUBLE 40dfffc000000000 LONG LONG 0000000000007fff 32767 +double_long_short_neg_32769 DOUBLE c0e0002000000000 LONG LONG ffffffffffff7fff -32769 +double_long_short_pos_32768 DOUBLE 40e0000000000000 LONG LONG 0000000000008000 32768 +double_long_int_neg_2147483648 DOUBLE c1e0000000000000 LONG LONG ffffffff80000000 -2147483648 +double_long_int_pos_2147483647 DOUBLE 41dfffffffc00000 LONG LONG 000000007fffffff 2147483647 +double_long_int_neg_2147483649 DOUBLE c1e0000000200000 LONG LONG ffffffff7fffffff -2147483649 +double_long_int_pos_2147483648 DOUBLE 41e0000000000000 LONG LONG 0000000080000000 2147483648 +double_long_long_neg_9223372036854776000 DOUBLE c3e0000000000000 LONG LONG 8000000000000000 NULL +double_long_long_pos_9223372036854776000 DOUBLE 43e0000000000000 LONG LONG +double_float_pos_zero DOUBLE 0000000000000000 FLOAT FLOAT 00000000 BITS:00000000 +double_float_neg_zero DOUBLE 8000000000000000 FLOAT FLOAT 80000000 BITS:80000000 +double_float_pos_one DOUBLE 3ff0000000000000 FLOAT FLOAT 3f800000 BITS:3f800000 +double_float_neg_one DOUBLE bff0000000000000 FLOAT FLOAT bf800000 BITS:bf800000 +double_float_fraction DOUBLE 3ff8000000000000 FLOAT FLOAT 3fc00000 BITS:3fc00000 +double_float_pos_inf DOUBLE 7ff0000000000000 FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_neg_inf DOUBLE fff0000000000000 FLOAT FLOAT ff800000 BITS:ff800000 +double_float_nan_payload DOUBLE 7ff8000000000001 FLOAT FLOAT NULL +double_float_nan_negative DOUBLE fff8000000000042 FLOAT FLOAT NULL +double_float_min_subnormal DOUBLE 0000000000000001 FLOAT FLOAT 00000000 BITS:00000000 +double_float_max_finite DOUBLE 7fefffffffffffff FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_byte_neg_128 DOUBLE c060000000000000 FLOAT FLOAT c3000000 BITS:c3000000 +double_float_byte_pos_127 DOUBLE 405fc00000000000 FLOAT FLOAT 42fe0000 BITS:42fe0000 +double_float_byte_neg_129 DOUBLE c060200000000000 FLOAT FLOAT c3010000 BITS:c3010000 +double_float_byte_pos_128 DOUBLE 4060000000000000 FLOAT FLOAT 43000000 BITS:43000000 +double_float_short_neg_32768 DOUBLE c0e0000000000000 FLOAT FLOAT c7000000 BITS:c7000000 +double_float_short_pos_32767 DOUBLE 40dfffc000000000 FLOAT FLOAT 46fffe00 BITS:46fffe00 +double_float_short_neg_32769 DOUBLE c0e0002000000000 FLOAT FLOAT c7000100 BITS:c7000100 +double_float_short_pos_32768 DOUBLE 40e0000000000000 FLOAT FLOAT 47000000 BITS:47000000 +double_float_int_neg_2147483648 DOUBLE c1e0000000000000 FLOAT FLOAT cf000000 BITS:cf000000 +double_float_int_pos_2147483647 DOUBLE 41dfffffffc00000 FLOAT FLOAT 4f000000 BITS:4f000000 +double_float_int_neg_2147483649 DOUBLE c1e0000000200000 FLOAT FLOAT cf000000 BITS:cf000000 +double_float_int_pos_2147483648 DOUBLE 41e0000000000000 FLOAT FLOAT 4f000000 BITS:4f000000 +double_float_long_neg_9223372036854776000 DOUBLE c3e0000000000000 FLOAT FLOAT df000000 BITS:df000000 +double_float_long_pos_9223372036854776000 DOUBLE 43e0000000000000 FLOAT FLOAT 5f000000 BITS:5f000000 +double_double_pos_zero DOUBLE 0000000000000000 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 +double_double_neg_zero DOUBLE 8000000000000000 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 +double_double_pos_one DOUBLE 3ff0000000000000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 +double_double_neg_one DOUBLE bff0000000000000 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 +double_double_fraction DOUBLE 3ff8000000000000 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 +double_double_pos_inf DOUBLE 7ff0000000000000 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 +double_double_neg_inf DOUBLE fff0000000000000 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 +double_double_nan_payload DOUBLE 7ff8000000000001 DOUBLE DOUBLE NULL +double_double_nan_negative DOUBLE fff8000000000042 DOUBLE DOUBLE NULL +double_double_min_subnormal DOUBLE 0000000000000001 DOUBLE DOUBLE 0000000000000001 BITS:0000000000000001 +double_double_max_finite DOUBLE 7fefffffffffffff DOUBLE DOUBLE 7fefffffffffffff BITS:7fefffffffffffff +double_double_byte_neg_128 DOUBLE c060000000000000 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 +double_double_byte_pos_127 DOUBLE 405fc00000000000 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 +double_double_byte_neg_129 DOUBLE c060200000000000 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 +double_double_byte_pos_128 DOUBLE 4060000000000000 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 +double_double_short_neg_32768 DOUBLE c0e0000000000000 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 +double_double_short_pos_32767 DOUBLE 40dfffc000000000 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 +double_double_short_neg_32769 DOUBLE c0e0002000000000 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 +double_double_short_pos_32768 DOUBLE 40e0000000000000 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 +double_double_int_neg_2147483648 DOUBLE c1e0000000000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +double_double_int_pos_2147483647 DOUBLE 41dfffffffc00000 DOUBLE DOUBLE 41dfffffffc00000 BITS:41dfffffffc00000 +double_double_int_neg_2147483649 DOUBLE c1e0000000200000 DOUBLE DOUBLE c1e0000000200000 BITS:c1e0000000200000 +double_double_int_pos_2147483648 DOUBLE 41e0000000000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +double_double_long_neg_9223372036854776000 DOUBLE c3e0000000000000 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 +double_double_long_pos_9223372036854776000 DOUBLE 43e0000000000000 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 +float_long_pos_below FLOAT 5effffff LONG LONG 7fffff8000000000 9223371487098961920 +float_long_pos_above FLOAT 5f000001 LONG LONG +float_long_neg_inside FLOAT deffffff LONG LONG 8000008000000000 -9223371487098961920 +float_long_neg_outside FLOAT df000001 LONG LONG +double_long_pos_below DOUBLE 43dfffffffffffff LONG LONG 7ffffffffffffc00 9223372036854774784 +double_long_pos_above DOUBLE 43e0000000000001 LONG LONG +double_long_neg_inside DOUBLE c3dfffffffffffff LONG LONG 8000000000000400 -9223372036854774784 +double_long_neg_outside DOUBLE c3e0000000000001 LONG LONG +float_int_pos_below FLOAT 4effffff INT INT 7fffff80 2147483520 +float_int_pos_at_2p31 FLOAT 4f000000 INT INT +float_int_neg_at_min FLOAT cf000000 INT INT 80000000 NULL +float_int_neg_outside FLOAT cf000001 INT INT +double_int_pos_at_max DOUBLE 41dfffffffc00000 INT INT 7fffffff 2147483647 +double_int_pos_outside DOUBLE 41e0000000000000 INT INT +double_int_neg_at_min DOUBLE c1e0000000000000 INT INT 80000000 NULL +double_int_neg_outside DOUBLE c1e0000000200000 INT INT +float_byte_negative_fraction FLOAT bf000000 BYTE BYTE +float_short_negative_fraction FLOAT bf000000 SHORT SHORT +float_int_negative_fraction FLOAT bf000000 INT INT +float_long_negative_fraction FLOAT bf000000 LONG LONG +float_float_negative_fraction FLOAT bf000000 FLOAT FLOAT bf000000 BITS:bf000000 +float_double_negative_fraction FLOAT bf000000 DOUBLE DOUBLE bfe0000000000000 BITS:bfe0000000000000 +double_byte_negative_fraction DOUBLE bfe0000000000000 BYTE BYTE +double_short_negative_fraction DOUBLE bfe0000000000000 SHORT SHORT +double_int_negative_fraction DOUBLE bfe0000000000000 INT INT +double_long_negative_fraction DOUBLE bfe0000000000000 LONG LONG +double_float_negative_fraction DOUBLE bfe0000000000000 FLOAT FLOAT bf000000 BITS:bf000000 +double_double_negative_fraction DOUBLE bfe0000000000000 DOUBLE DOUBLE bfe0000000000000 BITS:bfe0000000000000 +float_byte_signaling_nan FLOAT 7f800001 BYTE BYTE 0 +float_short_signaling_nan FLOAT 7f800001 SHORT SHORT 0 +float_int_signaling_nan FLOAT 7f800001 INT INT NULL +float_long_signaling_nan FLOAT 7f800001 LONG LONG NULL +float_float_signaling_nan FLOAT 7f800001 FLOAT FLOAT NULL +float_double_signaling_nan FLOAT 7f800001 DOUBLE DOUBLE NULL +double_byte_signaling_nan DOUBLE 7ff0000000000001 BYTE BYTE 0 +double_short_signaling_nan DOUBLE 7ff0000000000001 SHORT SHORT 0 +double_int_signaling_nan DOUBLE 7ff0000000000001 INT INT NULL +double_long_signaling_nan DOUBLE 7ff0000000000001 LONG LONG NULL +double_float_signaling_nan DOUBLE 7ff0000000000001 FLOAT FLOAT NULL +double_double_signaling_nan DOUBLE 7ff0000000000001 DOUBLE DOUBLE NULL +double_float_half_min_subnormal DOUBLE 3690000000000000 FLOAT FLOAT 00000000 BITS:00000000 +double_float_above_half_min_subnormal DOUBLE 3690000000000001 FLOAT FLOAT 00000001 BITS:00000001 +double_float_negative_half_min_subnormal DOUBLE b690000000000000 FLOAT FLOAT 80000000 BITS:80000000 +double_float_float_min_normal DOUBLE 3810000000000000 FLOAT FLOAT 00800000 BITS:00800000 +double_float_below_float_min_normal DOUBLE 380fffffffffffff FLOAT FLOAT 00800000 BITS:00800000 +double_float_float_overflow_tie DOUBLE 47effffff0000000 FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_below_float_overflow DOUBLE 47efffffefffffff FLOAT FLOAT 7f7fffff BITS:7f7fffff +double_float_tie_one_even_down DOUBLE 3ff0000010000000 FLOAT FLOAT 3f800000 BITS:3f800000 +double_float_tie_one_even_up DOUBLE 3ff0000030000000 FLOAT FLOAT 3f800002 BITS:3f800002 +double_float_subnormal_normal_tie DOUBLE 380fffffe0000000 FLOAT FLOAT 00800000 BITS:00800000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv new file mode 100644 index 000000000..cdb47cb18 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv @@ -0,0 +1,49 @@ +# case_id input_type input_raw_bits target_type expected_wire_type expected_utf8_hex expected_sql_value +float_string_pos_zero FLOAT 00000000 STRING VARCHAR 302e30 0.0 +float_string_neg_zero FLOAT 80000000 STRING VARCHAR 2d302e30 -0.0 +float_string_one FLOAT 3f800000 STRING VARCHAR 312e30 1.0 +float_string_point_one FLOAT 3dcccccd STRING VARCHAR 302e3130303030303030313439303131363132 0.10000000149011612 +float_string_min_subnormal FLOAT 00000001 STRING VARCHAR 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_string_max_finite FLOAT 7f7fffff STRING VARCHAR 332e34303238323334363633383532383836453338 3.4028234663852886E38 +float_string_pos_infinity FLOAT 7f800000 STRING VARCHAR 496e66696e697479 Infinity +float_string_neg_infinity FLOAT ff800000 STRING VARCHAR 2d496e66696e697479 -Infinity +float_string_qnan FLOAT 7fc00002 STRING VARCHAR +float_string_snan_negative FLOAT ff800001 STRING VARCHAR +float_string_1e23 FLOAT 65a96816 STRING VARCHAR 392e393939393939373738313936333038453232 9.999999778196308E22 +float_varchar_pos_zero FLOAT 00000000 VARCHAR VARCHAR 302e30 0.0 +float_varchar_neg_zero FLOAT 80000000 VARCHAR VARCHAR 2d302e30 -0.0 +float_varchar_point_one FLOAT 3dcccccd VARCHAR VARCHAR 302e3130303030303030313439303131363132 0.10000000149011612 +float_varchar_min_subnormal FLOAT 00000001 VARCHAR VARCHAR 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_varchar_pos_infinity FLOAT 7f800000 VARCHAR VARCHAR 496e66696e697479 Infinity +float_varchar_qnan FLOAT 7fc00002 VARCHAR VARCHAR +float_symbol_pos_zero FLOAT 00000000 SYMBOL SYMBOL 302e30 0.0 +float_symbol_neg_zero FLOAT 80000000 SYMBOL SYMBOL 2d302e30 -0.0 +float_symbol_point_one FLOAT 3dcccccd SYMBOL SYMBOL 302e3130303030303030313439303131363132 0.10000000149011612 +float_symbol_min_subnormal FLOAT 00000001 SYMBOL SYMBOL 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_symbol_neg_infinity FLOAT ff800000 SYMBOL SYMBOL 2d496e66696e697479 -Infinity +float_symbol_snan_negative FLOAT ff800001 SYMBOL SYMBOL +double_string_pos_zero DOUBLE 0000000000000000 STRING VARCHAR 302e30 0.0 +double_string_neg_zero DOUBLE 8000000000000000 STRING VARCHAR 2d302e30 -0.0 +double_string_one DOUBLE 3ff0000000000000 STRING VARCHAR 312e30 1.0 +double_string_neg_two_half DOUBLE c004000000000000 STRING VARCHAR 2d322e35 -2.5 +double_string_1e23 DOUBLE 44b52d02c7e14af6 STRING VARCHAR 312e30453233 1.0E23 +double_string_min_subnormal DOUBLE 0000000000000001 STRING VARCHAR 352e30452d333234 5.0E-324 +double_string_max_finite DOUBLE 7fefffffffffffff STRING VARCHAR 312e3739373639333133343836323331353745333038 1.7976931348623157E308 +double_string_pos_infinity DOUBLE 7ff0000000000000 STRING VARCHAR 496e66696e697479 Infinity +double_string_neg_infinity DOUBLE fff0000000000000 STRING VARCHAR 2d496e66696e697479 -Infinity +double_string_qnan DOUBLE 7ff8000000000002 STRING VARCHAR +double_string_snan_negative DOUBLE fff0000000000001 STRING VARCHAR +double_varchar_pos_zero DOUBLE 0000000000000000 VARCHAR VARCHAR 302e30 0.0 +double_varchar_neg_zero DOUBLE 8000000000000000 VARCHAR VARCHAR 2d302e30 -0.0 +double_varchar_neg_two_half DOUBLE c004000000000000 VARCHAR VARCHAR 2d322e35 -2.5 +double_varchar_1e23 DOUBLE 44b52d02c7e14af6 VARCHAR VARCHAR 312e30453233 1.0E23 +double_varchar_min_subnormal DOUBLE 0000000000000001 VARCHAR VARCHAR 352e30452d333234 5.0E-324 +double_varchar_pos_infinity DOUBLE 7ff0000000000000 VARCHAR VARCHAR 496e66696e697479 Infinity +double_varchar_qnan DOUBLE 7ff8000000000002 VARCHAR VARCHAR +double_symbol_pos_zero DOUBLE 0000000000000000 SYMBOL SYMBOL 302e30 0.0 +double_symbol_neg_zero DOUBLE 8000000000000000 SYMBOL SYMBOL 2d302e30 -0.0 +double_symbol_neg_two_half DOUBLE c004000000000000 SYMBOL SYMBOL 2d322e35 -2.5 +double_symbol_1e23 DOUBLE 44b52d02c7e14af6 SYMBOL SYMBOL 312e30453233 1.0E23 +double_symbol_max_finite DOUBLE 7fefffffffffffff SYMBOL SYMBOL 312e3739373639333133343836323331353745333038 1.7976931348623157E308 +double_symbol_neg_infinity DOUBLE fff0000000000000 SYMBOL SYMBOL 2d496e66696e697479 -Infinity +double_symbol_snan_negative DOUBLE fff0000000000001 SYMBOL SYMBOL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv new file mode 100644 index 000000000..ecf131bd5 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv @@ -0,0 +1,40 @@ +# case_id input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +8_positive_max 9 DECIMAL8 2 1 VALUE 000000000000005a - - - 9.0 +8_positive_over 10 DECIMAL8 2 1 INVALID - - - - - +8_negative_max -9 DECIMAL8 2 1 VALUE ffffffffffffffa6 - - - -9.0 +8_negative_over -10 DECIMAL8 2 1 INVALID - - - - - +16_positive_max 99 DECIMAL16 4 2 VALUE 00000000000026ac - - - 99.00 +16_positive_over 100 DECIMAL16 4 2 INVALID - - - - - +16_negative_max -99 DECIMAL16 4 2 VALUE ffffffffffffd954 - - - -99.00 +16_negative_over -100 DECIMAL16 4 2 INVALID - - - - - +32_positive_max 999999 DECIMAL32 9 3 VALUE 000000003b9ac618 - - - 999999.000 +32_positive_over 1000000 DECIMAL32 9 3 INVALID - - - - - +32_negative_max -999999 DECIMAL32 9 3 VALUE ffffffffc46539e8 - - - -999999.000 +32_negative_over -1000000 DECIMAL32 9 3 INVALID - - - - - +64_positive_max 99999999999999999 DECIMAL64 18 1 VALUE 0de0b6b3a763fff6 - - - 99999999999999999.0 +64_positive_over 100000000000000000 DECIMAL64 18 1 INVALID - - - - - +64_negative_max -99999999999999999 DECIMAL64 18 1 VALUE f21f494c589c000a - - - -99999999999999999.0 +64_negative_over -100000000000000000 DECIMAL64 18 1 INVALID - - - - - +128_positive_max 999999999999999999 DECIMAL128 38 20 VALUE 9dc2c4129cf00000 4b3b4ca85a86c474 - - 999999999999999999.00000000000000000000 +128_positive_over 1000000000000000000 DECIMAL128 38 20 INVALID - - - - - +128_negative_max -999999999999999999 DECIMAL128 38 20 VALUE 623d3bed63100000 b4c4b357a5793b8b - - -999999999999999999.00000000000000000000 +128_negative_over -1000000000000000000 DECIMAL128 38 20 INVALID - - - - - +256_positive_max 999999999999999999 DECIMAL256 75 57 VALUE b600000000000000 9ff45edd38440d7c 8b27fbc1cea32b80 0235fadd81c2822b 999999999999999999.000000000000000000000000000000000000000000000000000000000 +256_positive_over 1000000000000000000 DECIMAL256 75 57 INVALID - - - - - +256_negative_max -999999999999999999 DECIMAL256 75 57 VALUE 4a00000000000000 600ba122c7bbf283 74d8043e315cd47f fdca05227e3d7dd4 -999999999999999999.000000000000000000000000000000000000000000000000000000000 +256_negative_over -1000000000000000000 DECIMAL256 75 57 INVALID - - - - - +long_max_to_128 9223372036854775807 DECIMAL128 38 19 VALUE 7538dcfb76180000 4563918244f3ffff - - 9223372036854775807.0000000000000000000 +long_min_plus_one_to_128 -9223372036854775807 DECIMAL128 38 19 VALUE 8ac7230489e80000 ba9c6e7dbb0c0000 - - -9223372036854775807.0000000000000000000 +long_max_to_256 9223372036854775807 DECIMAL256 76 57 VALUE b600000000000000 b90234ab79b5257c cd3668f47a8b47cf 14643e5ae44d12b8 9223372036854775807.000000000000000000000000000000000000000000000000000000000 +long_min_plus_one_to_256 -9223372036854775807 DECIMAL256 76 57 VALUE 4a00000000000000 46fdcb54864ada83 32c9970b8574b830 eb9bc1a51bb2ed47 -9223372036854775807.000000000000000000000000000000000000000000000000000000000 +long_max_256_storage_over 9223372036854775807 DECIMAL256 76 58 INVALID - - - - - +long_min_plus_one_256_storage_over -9223372036854775807 DECIMAL256 76 58 INVALID - - - - - +zero_64_max_scale 0 DECIMAL64 18 18 VALUE 0000000000000000 - - - 0.000000000000000000 +zero_128_max_scale 0 DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +zero_256_max_scale 0 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +long_min_null_8 -9223372036854775808 DECIMAL8 2 1 NULL - - - - NULL +long_min_null_16 -9223372036854775808 DECIMAL16 4 2 NULL - - - - NULL +long_min_null_32 -9223372036854775808 DECIMAL32 9 3 NULL - - - - NULL +long_min_null_64 -9223372036854775808 DECIMAL64 18 1 NULL - - - - NULL +long_min_null_128 -9223372036854775808 DECIMAL128 38 20 NULL - - - - NULL +long_min_null_256 -9223372036854775808 DECIMAL256 76 58 NULL - - - - NULL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv new file mode 100644 index 000000000..53af275f1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv @@ -0,0 +1,49 @@ +# case_id input target expected +# means the target's missing-value representation; BYTE/SHORT materialize it as zero. +long_min_byte -9223372036854775808 BYTE +long_min_short -9223372036854775808 SHORT +long_min_int -9223372036854775808 INT +long_min_long -9223372036854775808 LONG +long_min_float -9223372036854775808 FLOAT +long_min_double -9223372036854775808 DOUBLE +byte_min -128 BYTE -128 +byte_max 127 BYTE 127 +byte_zero 0 BYTE 0 +byte_below -129 BYTE +byte_above 128 BYTE +short_min -32768 SHORT -32768 +short_max 32767 SHORT 32767 +short_zero 0 SHORT 0 +short_below -32769 SHORT +short_above 32768 SHORT +int_min -2147483648 INT +int_min_plus_one -2147483647 INT -2147483647 +int_max 2147483647 INT 2147483647 +int_zero 0 INT 0 +int_below -2147483649 INT +int_above 2147483648 INT +zero_long 0 LONG 0 +minus_one_long -1 LONG -1 +long_max 9223372036854775807 LONG 9223372036854775807 +long_min_plus_one -9223372036854775807 LONG -9223372036854775807 +zero_float 0 FLOAT 0x00000000 +minus_one_float -1 FLOAT 0xbf800000 +precision_float 2147483647 FLOAT 0x4f000000 +float_exact_below 16777215 FLOAT 0x4b7fffff +float_exact_boundary 16777216 FLOAT 0x4b800000 +float_round_down 16777217 FLOAT 0x4b800000 +float_round_even 16777218 FLOAT 0x4b800001 +float_round_up 16777219 FLOAT 0x4b800002 +float_negative_round -16777217 FLOAT 0xcb800000 +long_min_plus_one_float -9223372036854775807 FLOAT 0xdf000000 +long_max_float 9223372036854775807 FLOAT 0x5f000000 +zero_double 0 DOUBLE 0x0000000000000000 +minus_one_double -1 DOUBLE 0xbff0000000000000 +precision_double 9007199254740993 DOUBLE 0x4340000000000000 +double_exact_below 9007199254740991 DOUBLE 0x433fffffffffffff +double_exact_boundary 9007199254740992 DOUBLE 0x4340000000000000 +double_next_even 9007199254740994 DOUBLE 0x4340000000000001 +double_round_up 9007199254740995 DOUBLE 0x4340000000000002 +double_negative_round -9007199254740993 DOUBLE 0xc340000000000000 +long_min_plus_one_double -9223372036854775807 DOUBLE 0xc3e0000000000000 +long_max_double 9223372036854775807 DOUBLE 0x43e0000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv new file mode 100644 index 000000000..23967eb15 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv @@ -0,0 +1,19 @@ +# case_id input_long target_type expected_wire_type expected_utf8_hex expected_sql_value +long_string_min -9223372036854775808 STRING VARCHAR +long_string_min_plus_1 -9223372036854775807 STRING VARCHAR 2d39323233333732303336383534373735383037 -9223372036854775807 +long_string_neg_1 -1 STRING VARCHAR 2d31 -1 +long_string_zero 0 STRING VARCHAR 30 0 +long_string_one 1 STRING VARCHAR 31 1 +long_string_max 9223372036854775807 STRING VARCHAR 39323233333732303336383534373735383037 9223372036854775807 +long_varchar_min -9223372036854775808 VARCHAR VARCHAR +long_varchar_min_plus_1 -9223372036854775807 VARCHAR VARCHAR 2d39323233333732303336383534373735383037 -9223372036854775807 +long_varchar_neg_1 -1 VARCHAR VARCHAR 2d31 -1 +long_varchar_zero 0 VARCHAR VARCHAR 30 0 +long_varchar_one 1 VARCHAR VARCHAR 31 1 +long_varchar_max 9223372036854775807 VARCHAR VARCHAR 39323233333732303336383534373735383037 9223372036854775807 +long_symbol_min -9223372036854775808 SYMBOL SYMBOL +long_symbol_min_plus_1 -9223372036854775807 SYMBOL SYMBOL 2d39323233333732303336383534373735383037 -9223372036854775807 +long_symbol_neg_1 -1 SYMBOL SYMBOL 2d31 -1 +long_symbol_zero 0 SYMBOL SYMBOL 30 0 +long_symbol_one 1 SYMBOL SYMBOL 31 1 +long_symbol_max 9223372036854775807 SYMBOL SYMBOL 39323233333732303336383534373735383037 9223372036854775807 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv new file mode 100644 index 000000000..cdd0db95a --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv @@ -0,0 +1,13 @@ +# case_id input_value target_type expected_wire_type expected_wire_value expected_sql_null +long_timestamp_min -9223372036854775808 TIMESTAMP TIMESTAMP true +long_timestamp_min_plus_one -9223372036854775807 TIMESTAMP TIMESTAMP -9223372036854775807 false +long_timestamp_negative_one -1 TIMESTAMP TIMESTAMP -1 false +long_timestamp_zero 0 TIMESTAMP TIMESTAMP 0 false +long_timestamp_one 1 TIMESTAMP TIMESTAMP 1 false +long_timestamp_max 9223372036854775807 TIMESTAMP TIMESTAMP 9223372036854775807 false +long_timestamp_nanos_min -9223372036854775808 TIMESTAMP_NANOS TIMESTAMP_NANOS true +long_timestamp_nanos_min_plus_one -9223372036854775807 TIMESTAMP_NANOS TIMESTAMP_NANOS -9223372036854775807 false +long_timestamp_nanos_negative_one -1 TIMESTAMP_NANOS TIMESTAMP_NANOS -1 false +long_timestamp_nanos_zero 0 TIMESTAMP_NANOS TIMESTAMP_NANOS 0 false +long_timestamp_nanos_one 1 TIMESTAMP_NANOS TIMESTAMP_NANOS 1 false +long_timestamp_nanos_max 9223372036854775807 TIMESTAMP_NANOS TIMESTAMP_NANOS 9223372036854775807 false diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv new file mode 100644 index 000000000..caa4173ee --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv @@ -0,0 +1,61 @@ +# case_id source_type source_scale source_ll_hex source_lh_hex source_hl_hex source_hh_hex target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +64_to_8 DECIMAL64 1 000000000000000c - - - DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +64_to_16 DECIMAL64 1 000000000000000c - - - DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +64_to_32 DECIMAL64 1 000000000000000c - - - DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +64_to_64 DECIMAL64 1 000000000000000c - - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +64_to_128 DECIMAL64 1 000000000000000c - - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +64_to_256 DECIMAL64 1 000000000000000c - - - DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +128_to_8 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +128_to_16 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +128_to_32 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +128_to_64 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +128_to_128 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +128_to_256 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +256_to_8 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +256_to_16 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +256_to_32 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +256_to_64 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +256_to_128 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +256_to_256 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +64_zero DECIMAL64 2 0000000000000000 - - - DECIMAL64 18 2 VALUE 0000000000000000 - - - 0.00 +64_negative DECIMAL64 2 fffffffffffffff4 - - - DECIMAL64 18 2 VALUE fffffffffffffff4 - - - -0.12 +64_upscale DECIMAL64 1 000000000000000c - - - DECIMAL64 18 3 VALUE 00000000000004b0 - - - 1.200 +64_downscale_exact DECIMAL64 3 00000000000004b0 - - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +64_downscale_inexact DECIMAL64 3 00000000000004b1 - - - DECIMAL64 18 1 INVALID - - - - - +128_zero DECIMAL128 2 0000000000000000 0000000000000000 - - DECIMAL128 38 2 VALUE 0000000000000000 0000000000000000 - - 0.00 +128_negative DECIMAL128 2 fffffffffffffff4 ffffffffffffffff - - DECIMAL128 38 2 VALUE fffffffffffffff4 ffffffffffffffff - - -0.12 +128_upscale DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL128 38 3 VALUE 00000000000004b0 0000000000000000 - - 1.200 +128_downscale_exact DECIMAL128 3 00000000000004b0 0000000000000000 - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +128_downscale_inexact DECIMAL128 3 00000000000004b1 0000000000000000 - - DECIMAL128 38 1 INVALID - - - - - +256_zero DECIMAL256 2 0000000000000000 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 2 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.00 +256_negative DECIMAL256 2 fffffffffffffff4 ffffffffffffffff ffffffffffffffff ffffffffffffffff DECIMAL256 76 2 VALUE fffffffffffffff4 ffffffffffffffff ffffffffffffffff ffffffffffffffff -0.12 +256_upscale DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 3 VALUE 00000000000004b0 0000000000000000 0000000000000000 0000000000000000 1.200 +256_downscale_exact DECIMAL256 3 00000000000004b0 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +256_downscale_inexact DECIMAL256 3 00000000000004b1 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 INVALID - - - - - +8_precision_max DECIMAL64 0 0000000000000063 - - - DECIMAL8 2 0 VALUE 0000000000000063 - - - 99 +8_precision_overflow DECIMAL64 0 0000000000000064 - - - DECIMAL8 2 0 INVALID - - - - - +16_precision_max DECIMAL64 0 000000000000270f - - - DECIMAL16 4 0 VALUE 000000000000270f - - - 9999 +16_precision_overflow DECIMAL64 0 0000000000002710 - - - DECIMAL16 4 0 INVALID - - - - - +32_precision_max DECIMAL64 0 000000003b9ac9ff - - - DECIMAL32 9 0 VALUE 000000003b9ac9ff - - - 999999999 +32_precision_overflow DECIMAL64 0 000000003b9aca00 - - - DECIMAL32 9 0 INVALID - - - - - +64_precision_max DECIMAL64 0 0de0b6b3a763ffff - - - DECIMAL64 18 0 VALUE 0de0b6b3a763ffff - - - 999999999999999999 +64_precision_overflow DECIMAL64 0 0de0b6b3a7640000 - - - DECIMAL64 18 0 INVALID - - - - - +128_precision_max DECIMAL128 0 098a223fffffffff 4b3b4ca85a86c47a - - DECIMAL128 38 0 VALUE 098a223fffffffff 4b3b4ca85a86c47a - - 99999999999999999999999999999999999999 +128_precision_overflow DECIMAL128 0 098a224000000000 4b3b4ca85a86c47a - - DECIMAL128 38 0 INVALID - - - - - +256_precision_max DECIMAL256 0 ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 DECIMAL256 76 0 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_precision_overflow DECIMAL256 0 0000000000000000 7775a5f171951000 0764b4abe8652979 161bcca7119915b5 DECIMAL256 76 0 INVALID - - - - - +8_precision_negative_max DECIMAL64 0 ffffffffffffff9d - - - DECIMAL8 2 0 VALUE ffffffffffffff9d - - - -99 +8_precision_negative_overflow DECIMAL64 0 ffffffffffffff9c - - - DECIMAL8 2 0 INVALID - - - - - +16_precision_negative_max DECIMAL64 0 ffffffffffffd8f1 - - - DECIMAL16 4 0 VALUE ffffffffffffd8f1 - - - -9999 +16_precision_negative_overflow DECIMAL64 0 ffffffffffffd8f0 - - - DECIMAL16 4 0 INVALID - - - - - +32_precision_negative_max DECIMAL64 0 ffffffffc4653601 - - - DECIMAL32 9 0 VALUE ffffffffc4653601 - - - -999999999 +32_precision_negative_overflow DECIMAL64 0 ffffffffc4653600 - - - DECIMAL32 9 0 INVALID - - - - - +64_precision_negative_max DECIMAL64 0 f21f494c589c0001 - - - DECIMAL64 18 0 VALUE f21f494c589c0001 - - - -999999999999999999 +64_precision_negative_overflow DECIMAL64 0 f21f494c589c0000 - - - DECIMAL64 18 0 INVALID - - - - - +128_precision_negative_max DECIMAL128 0 f675ddc000000001 b4c4b357a5793b85 - - DECIMAL128 38 0 VALUE f675ddc000000001 b4c4b357a5793b85 - - -99999999999999999999999999999999999999 +128_precision_negative_overflow DECIMAL128 0 f675ddc000000000 b4c4b357a5793b85 - - DECIMAL128 38 0 INVALID - - - - - +256_precision_negative_max DECIMAL256 0 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a DECIMAL256 76 0 VALUE 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a -9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_precision_negative_overflow DECIMAL256 0 0000000000000000 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a DECIMAL256 76 0 INVALID - - - - - +64_zero_max_scale DECIMAL64 18 0000000000000000 - - - DECIMAL64 18 18 VALUE 0000000000000000 - - - 0.000000000000000000 +128_zero_max_scale DECIMAL128 38 0000000000000000 0000000000000000 - - DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +256_zero_max_scale DECIMAL256 76 0000000000000000 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv new file mode 100644 index 000000000..40af5fb0f --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv @@ -0,0 +1,25 @@ +# case_id source_type source_scale source_ll_hex source_lh_hex source_hl_hex source_hh_hex target_type input_form expected_text +64_string_positive_trailing DECIMAL64 2 0000000000003034 - - - STRING VALUE 123.40 +64_varchar_negative_fraction DECIMAL64 3 ffffffffffffff83 - - - VARCHAR VALUE -0.125 +64_string_precision_max DECIMAL64 0 0de0b6b3a763ffff - - - STRING VALUE 999999999999999999 +64_varchar_negative_max_scale DECIMAL64 18 f21f494c589c0001 - - - VARCHAR VALUE -0.999999999999999999 +64_string_leading_fraction DECIMAL64 18 0000000000000001 - - - STRING VALUE 0.000000000000000001 +64_varchar_zero_max_scale DECIMAL64 18 0000000000000000 - - - VARCHAR VALUE 0.000000000000000000 +128_string_positive_trailing DECIMAL128 2 0000000000003034 0000000000000000 - - STRING VALUE 123.40 +128_varchar_negative_fraction DECIMAL128 3 ffffffffffffff83 ffffffffffffffff - - VARCHAR VALUE -0.125 +128_string_precision_max DECIMAL128 0 098a223fffffffff 4b3b4ca85a86c47a - - STRING VALUE 99999999999999999999999999999999999999 +128_varchar_negative_max_scale DECIMAL128 38 f675ddc000000001 b4c4b357a5793b85 - - VARCHAR VALUE -0.99999999999999999999999999999999999999 +128_string_leading_fraction DECIMAL128 38 0000000000000001 0000000000000000 - - STRING VALUE 0.00000000000000000000000000000000000001 +128_varchar_zero_max_scale DECIMAL128 38 0000000000000000 0000000000000000 - - VARCHAR VALUE 0.00000000000000000000000000000000000000 +256_string_positive_trailing DECIMAL256 2 0000000000003034 0000000000000000 0000000000000000 0000000000000000 STRING VALUE 123.40 +256_varchar_negative_fraction DECIMAL256 3 ffffffffffffff83 ffffffffffffffff ffffffffffffffff ffffffffffffffff VARCHAR VALUE -0.125 +256_string_precision_max DECIMAL256 0 ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 STRING VALUE 9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_varchar_negative_max_scale DECIMAL256 76 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a VARCHAR VALUE -0.9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_string_leading_fraction DECIMAL256 76 0000000000000001 0000000000000000 0000000000000000 0000000000000000 STRING VALUE 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +256_varchar_zero_max_scale DECIMAL256 76 0000000000000000 0000000000000000 0000000000000000 0000000000000000 VARCHAR VALUE 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +64_string_java_null DECIMAL64 0 - - - - STRING JAVA_NULL - +64_varchar_null_value DECIMAL64 0 - - - - VARCHAR NULL_VALUE - +128_string_java_null DECIMAL128 0 - - - - STRING JAVA_NULL - +128_varchar_null_value DECIMAL128 0 - - - - VARCHAR NULL_VALUE - +256_string_java_null DECIMAL256 0 - - - - STRING JAVA_NULL - +256_varchar_null_value DECIMAL256 0 - - - - VARCHAR NULL_VALUE - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv new file mode 100644 index 000000000..0b31a9fdd --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv @@ -0,0 +1,19 @@ +# case_id input bits outcome value_hex +bit1_max z 1 VALUE 1 +bit7_max zz 7 VALUE 7f +bit8_max zz 8 VALUE ff +bit15_max zzz 15 VALUE 7fff +bit16_max zzzz 16 VALUE ffff +bit31_max zzzzzzz 31 VALUE 7fffffff +bit32_max zzzzzzz 32 VALUE ffffffff +bit60_max zzzzzzzzzzzz 60 VALUE fffffffffffffff +uppercase U 5 VALUE 1a +zero 0 5 VALUE 0 +whole_prefix_consumed 10 5 VALUE 1 +suffix_after_cap zzzzzzzzzzzz! 60 VALUE fffffffffffffff +unicode_after_cap zzzzzzzzzzzzé 60 VALUE fffffffffffffff +source_null 8 NULL +empty 8 NULL +too_short zzzzzzzzzzz 60 INVALID +invalid_consumed a 5 INVALID +unicode_consumed é 5 INVALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv new file mode 100644 index 000000000..943b815d9 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv @@ -0,0 +1,24 @@ +# case_id input outcome l0_hex l1_hex l2_hex l3_hex +zero 0x00 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 +byte_max 0xff VALUE 00000000000000ff 0000000000000000 0000000000000000 0000000000000000 +uppercase_digits 0xABCD VALUE 000000000000abcd 0000000000000000 0000000000000000 0000000000000000 +leading_zeroes 0x0001 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 +two_limbs 0x0123456789abcdeffedcba9876543210 VALUE fedcba9876543210 0123456789abcdef 0000000000000000 0000000000000000 +four_limbs 0x0123456789abcdeffedcba9876543210112233445566778899aabbccddeeff00 VALUE 99aabbccddeeff00 1122334455667788 fedcba9876543210 0123456789abcdef +all_ones 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff VALUE ffffffffffffffff ffffffffffffffff ffffffffffffffff ffffffffffffffff +single_min_limb 0x8000000000000000 VALUE 8000000000000000 0000000000000000 0000000000000000 0000000000000000 +partial_high_group 0x00112233445566778899 VALUE 2233445566778899 0000000000000011 0000000000000000 0000000000000000 +three_limbs 0x111111111111111122222222222222223333333333333333 VALUE 3333333333333333 2222222222222222 1111111111111111 0000000000000000 +source_null NULL +empty_string INVALID +empty_digits 0x INVALID +odd_digits 0x0 INVALID +uppercase_prefix 0X00 INVALID +leading_sign -0x00 INVALID +leading_space 0x00 INVALID +internal_underscore 0x0_0 INVALID +non_hex 0xgg INVALID +non_ascii 0xéé INVALID +trailing_space 0x00 INVALID +too_long 0x000000000000000000000000000000000000000000000000000000000000000000 INVALID +null_sentinel 0x8000000000000000800000000000000080000000000000008000000000000000 INVALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv new file mode 100644 index 000000000..9be92bbd1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv @@ -0,0 +1,81 @@ +# case_id input_utf16_hex expected_wire_value expected_sql_value +true_case_00 0074007200750065 1 true +true_case_01 0054007200750065 1 true +true_case_02 0074005200750065 1 true +true_case_03 0054005200750065 1 true +true_case_04 0074007200550065 1 true +true_case_05 0054007200550065 1 true +true_case_06 0074005200550065 1 true +true_case_07 0054005200550065 1 true +true_case_08 0074007200750045 1 true +true_case_09 0054007200750045 1 true +true_case_10 0074005200750045 1 true +true_case_11 0054005200750045 1 true +true_case_12 0074007200550045 1 true +true_case_13 0054007200550045 1 true +true_case_14 0074005200550045 1 true +true_case_15 0054005200550045 1 true +false_case_00 00660061006c00730065 0 false +false_case_01 00460061006c00730065 0 false +false_case_02 00660041006c00730065 0 false +false_case_03 00460041006c00730065 0 false +false_case_04 00660061004c00730065 0 false +false_case_05 00460061004c00730065 0 false +false_case_06 00660041004c00730065 0 false +false_case_07 00460041004c00730065 0 false +false_case_08 00660061006c00530065 0 false +false_case_09 00460061006c00530065 0 false +false_case_10 00660041006c00530065 0 false +false_case_11 00460041006c00530065 0 false +false_case_12 00660061004c00530065 0 false +false_case_13 00460061004c00530065 0 false +false_case_14 00660041004c00530065 0 false +false_case_15 00460041004c00530065 0 false +false_case_16 00660061006c00730045 0 false +false_case_17 00460061006c00730045 0 false +false_case_18 00660041006c00730045 0 false +false_case_19 00460041006c00730045 0 false +false_case_20 00660061004c00730045 0 false +false_case_21 00460061004c00730045 0 false +false_case_22 00660041004c00730045 0 false +false_case_23 00460041004c00730045 0 false +false_case_24 00660061006c00530045 0 false +false_case_25 00460061006c00530045 0 false +false_case_26 00660041006c00530045 0 false +false_case_27 00460041006c00530045 0 false +false_case_28 00660061004c00530045 0 false +false_case_29 00460061004c00530045 0 false +false_case_30 00660041004c00530045 0 false +false_case_31 00460041004c00530045 0 false +digit_zero 0030 0 false +digit_one 0031 1 true +explicit_null false +empty +leading_space 00200074007200750065 +trailing_space 00740072007500650020 +leading_tab 00090074007200750065 +trailing_lf 0074007200750065000a +double_zero 00300030 +zero_one 00300031 +one_zero 00310030 +plus_one 002b0031 +minus_zero 002d0030 +yes 007900650073 +truth 00740072007500740068 +falsehood 00660061006c007300650068006f006f0064 +unicode_capital_t 0166007200750065 +fullwidth_one ff11 +nul_true 00000074007200750065 +lone_high d800 +lone_low dc00 +true_nul 00740072007500650000 +single_t 0074 +single_f 0066 +no 006e006f +null_word 006e0075006c006c +digit_two 0032 +minus_one 002d0031 +true_question 0074007200750065003f +spaces_only 00200020 +false_long_s 00660061006c017f0065 +true_embedded_high 00740072d8000065 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv new file mode 100644 index 000000000..d2d8231da --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv @@ -0,0 +1,56 @@ +# case_id input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +8_precision_max 9.9 DECIMAL8 2 1 VALUE 0000000000000063 - - - 9.9 +8_precision_over 10.0 DECIMAL8 2 1 INVALID - - - - - +8_precision_loss 0.01 DECIMAL8 2 1 INVALID - - - - - +8_java_null DECIMAL8 2 1 NULL - - - - NULL +16_precision_max 99.99 DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +16_precision_over 100.00 DECIMAL16 4 2 INVALID - - - - - +16_precision_loss 0.001 DECIMAL16 4 2 INVALID - - - - - +16_java_null DECIMAL16 4 2 NULL - - - - NULL +32_precision_max 999999.999 DECIMAL32 9 3 VALUE 000000003b9ac9ff - - - 999999.999 +32_precision_over 1000000.000 DECIMAL32 9 3 INVALID - - - - - +32_precision_loss 0.0001 DECIMAL32 9 3 INVALID - - - - - +32_java_null DECIMAL32 9 3 NULL - - - - NULL +64_precision_max 99999999999999999.9 DECIMAL64 18 1 VALUE 0de0b6b3a763ffff - - - 99999999999999999.9 +64_precision_over 100000000000000000.0 DECIMAL64 18 1 INVALID - - - - - +64_precision_loss 0.01 DECIMAL64 18 1 INVALID - - - - - +64_java_null DECIMAL64 18 1 NULL - - - - NULL +128_precision_max 999999999999999999.99999999999999999999 DECIMAL128 38 20 VALUE 098a223fffffffff 4b3b4ca85a86c47a - - 999999999999999999.99999999999999999999 +128_precision_over 1000000000000000000.0 DECIMAL128 38 20 INVALID - - - - - +128_precision_loss 0.000000000000000000001 DECIMAL128 38 20 INVALID - - - - - +128_java_null DECIMAL128 38 20 NULL - - - - NULL +256_precision_max 9999999999999999999.999999999999999999999999999999999999999999999999999999999 DECIMAL256 76 57 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 9999999999999999999.999999999999999999999999999999999999999999999999999999999 +256_precision_over 10000000000000000000 DECIMAL256 76 57 INVALID - - - - - +256_precision_loss 1e-58 DECIMAL256 76 57 INVALID - - - - - +256_java_null DECIMAL256 76 57 NULL - - - - NULL +8_leading_space_plus_zeros +00.5 DECIMAL8 2 1 VALUE 0000000000000005 - - - 0.5 +8_zero_exponent_after_leading_zeros 0000.e80 DECIMAL8 1 0 VALUE 0000000000000000 - - - 0 +8_full_scale_fraction .5 DECIMAL8 1 1 VALUE 0000000000000005 - - - 0.5 +8_nan NaN DECIMAL8 2 1 NULL - - - - NULL +8_empty DECIMAL8 2 1 INVALID - - - - - +16_point_without_fraction 123. DECIMAL16 4 1 VALUE 00000000000004ce - - - 123.0 +16_infinity Infinity DECIMAL16 4 1 NULL - - - - NULL +16_point_only . DECIMAL16 4 1 INVALID - - - - - +32_positive_exponent 1.23e3 DECIMAL32 9 2 VALUE 000000000001e078 - - - 1230.00 +32_positive_nan +NaN DECIMAL32 9 2 NULL - - - - NULL +32_multiple_points 1.2.3 DECIMAL32 9 2 INVALID - - - - - +64_negative_exponent -2.5e-2 DECIMAL64 18 3 VALUE ffffffffffffffe7 - - - -0.025 +64_incomplete_exponent 1e DECIMAL64 18 3 INVALID - - - - - +64_exponent_underscore 1e1_0 DECIMAL64 18 2 VALUE 000000e8d4a51000 - - - 10000000000.00 +64_nan_decimal_suffix NaNm DECIMAL64 18 3 NULL - - - - NULL +128_trailing_zeros_relaxed 1.2300 DECIMAL128 38 2 VALUE 000000000000007b 0000000000000000 - - 1.23 +64_double_suffix -7.5d DECIMAL64 18 2 VALUE fffffffffffffd12 - - - -7.50 +128_float_suffix 123.0f DECIMAL128 38 4 VALUE 000000000012c4b0 0000000000000000 - - 123.0000 +128_decimal_suffix_strict_loss 1.2300M DECIMAL128 38 2 INVALID - - - - - +128_float_then_decimal_suffix_strict_loss 1.2300fm DECIMAL128 38 2 INVALID - - - - - +128_decimal_then_float_suffix_invalid 1.2300mf DECIMAL128 38 2 INVALID - - - - - +128_nan NaN DECIMAL128 38 4 NULL - - - - NULL +128_nan_junk NaNjunk DECIMAL128 38 4 INVALID - - - - - +256_negative_infinity -Infinity DECIMAL256 76 8 NULL - - - - NULL +256_infinity_junk Infinitygarbage DECIMAL256 76 8 INVALID - - - - - +256_lowercase_nan nan DECIMAL256 76 8 INVALID - - - - - +256_upper_exponent 7.89E-3 DECIMAL256 76 5 VALUE 0000000000000315 0000000000000000 0000000000000000 0000000000000000 0.00789 +256_exponent_int_max 1e2147483647 DECIMAL256 76 0 INVALID - - - - - +256_exponent_int_min 1e-2147483648 DECIMAL256 76 0 INVALID - - - - - +256_full_scale_fraction 0.9999999999999999999999999999999999999999999999999999999999999999999999999999 DECIMAL256 76 76 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 0.9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_full_scale_zero 0 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv new file mode 100644 index 000000000..5f7c243bf --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv @@ -0,0 +1,433 @@ +# Seven fields: case_id, input_utf16_hex, target_type, expected_wire_type, expected_wire_bits, expected_sql_value, outcome +# Input is lowercase UTF-16 code-unit hex; empty is a present empty string and is explicit null. +# Wire bits are lowercase fixed-width hexadecimal. means bitmap null/no value bytes; paired fields mean local rejection. +# BITS: means stored floating raw bits. Parsed INT/LONG minima and floating NaN have present wire bits but SQL NULL. +diag_001 BYTE BYTE 0 VALID +diag_002 BYTE BYTE INVALID +diag_003 0030 BYTE BYTE 00 0 VALID +diag_004 002d0030 BYTE BYTE 00 0 VALID +diag_005 002b0030 BYTE BYTE 00 0 VALID +diag_006 00300030 BYTE BYTE 00 0 VALID +diag_007 0031 BYTE BYTE 01 1 VALID +diag_008 002b0031 BYTE BYTE 01 1 VALID +diag_009 002d0031 BYTE BYTE ff -1 VALID +diag_010 003100320037 BYTE BYTE 7f 127 VALID +diag_011 003100320038 BYTE BYTE INVALID +diag_012 002d003100320038 BYTE BYTE 80 -128 VALID +diag_013 002d003100320039 BYTE BYTE INVALID +diag_014 00330032003700360037 BYTE BYTE INVALID +diag_015 00330032003700360038 BYTE BYTE INVALID +diag_016 002d00330032003700360038 BYTE BYTE INVALID +diag_017 002d00330032003700360039 BYTE BYTE INVALID +diag_018 0032003100340037003400380033003600340037 BYTE BYTE INVALID +diag_019 0032003100340037003400380033003600340038 BYTE BYTE INVALID +diag_020 002d0032003100340037003400380033003600340038 BYTE BYTE INVALID +diag_021 002d0032003100340037003400380033003600340039 BYTE BYTE INVALID +diag_022 0039003200320033003300370032003000330036003800350034003700370035003800300037 BYTE BYTE INVALID +diag_023 0039003200320033003300370032003000330036003800350034003700370035003800300038 BYTE BYTE INVALID +diag_024 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 BYTE BYTE INVALID +diag_025 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 BYTE BYTE INVALID +diag_026 0031002e0030 BYTE BYTE INVALID +diag_027 0031002e0035 BYTE BYTE INVALID +diag_028 003100650032 BYTE BYTE INVALID +diag_029 00310065002d0032 BYTE BYTE INVALID +diag_030 0031005f0030 BYTE BYTE 0a 10 VALID +diag_031 002d0031005f0030 BYTE BYTE f6 -10 VALID +diag_032 005f0031 BYTE BYTE INVALID +diag_033 0031005f BYTE BYTE INVALID +diag_034 0031005f005f0030 BYTE BYTE INVALID +diag_035 0031007f0030 BYTE BYTE INVALID +diag_036 0031004c BYTE BYTE INVALID +diag_037 0031006c BYTE BYTE INVALID +diag_038 00200031 BYTE BYTE INVALID +diag_039 00310020 BYTE BYTE INVALID +diag_040 00090031 BYTE BYTE INVALID +diag_041 0031000a BYTE BYTE INVALID +diag_042 0030007800310030 BYTE BYTE INVALID +diag_043 004e0061004e BYTE BYTE INVALID +diag_044 006e0061006e BYTE BYTE INVALID +diag_045 0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_046 002b0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_047 002d0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_048 0069006e0066 BYTE BYTE INVALID +diag_049 00310065003300300039 BYTE BYTE INVALID +diag_050 00310065002d0034003000300030 BYTE BYTE INVALID +diag_051 00310038003400340036003700340034003000370033003700300039003500350031003600310035 BYTE BYTE INVALID +diag_052 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 BYTE BYTE INVALID +diag_053 0000 BYTE BYTE INVALID +diag_054 00000031 BYTE BYTE INVALID +diag_055 00310000 BYTE BYTE INVALID +diag_056 003100000032 BYTE BYTE INVALID +diag_057 0661 BYTE BYTE INVALID +diag_058 00a00031 BYTE BYTE INVALID +diag_059 d800 BYTE BYTE INVALID +diag_060 0031d800 BYTE BYTE INVALID +diag_061 dc00 BYTE BYTE INVALID +diag_062 SHORT SHORT 0 VALID +diag_063 SHORT SHORT INVALID +diag_064 0030 SHORT SHORT 0000 0 VALID +diag_065 002d0030 SHORT SHORT 0000 0 VALID +diag_066 002b0030 SHORT SHORT INVALID +diag_067 00300030 SHORT SHORT 0000 0 VALID +diag_068 0031 SHORT SHORT 0001 1 VALID +diag_069 002b0031 SHORT SHORT INVALID +diag_070 002d0031 SHORT SHORT ffff -1 VALID +diag_071 003100320037 SHORT SHORT 007f 127 VALID +diag_072 003100320038 SHORT SHORT 0080 128 VALID +diag_073 002d003100320038 SHORT SHORT ff80 -128 VALID +diag_074 002d003100320039 SHORT SHORT ff7f -129 VALID +diag_075 00330032003700360037 SHORT SHORT 7fff 32767 VALID +diag_076 00330032003700360038 SHORT SHORT INVALID +diag_077 002d00330032003700360038 SHORT SHORT 8000 -32768 VALID +diag_078 002d00330032003700360039 SHORT SHORT INVALID +diag_079 0032003100340037003400380033003600340037 SHORT SHORT INVALID +diag_080 0032003100340037003400380033003600340038 SHORT SHORT INVALID +diag_081 002d0032003100340037003400380033003600340038 SHORT SHORT INVALID +diag_082 002d0032003100340037003400380033003600340039 SHORT SHORT INVALID +diag_083 0039003200320033003300370032003000330036003800350034003700370035003800300037 SHORT SHORT INVALID +diag_084 0039003200320033003300370032003000330036003800350034003700370035003800300038 SHORT SHORT INVALID +diag_085 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 SHORT SHORT INVALID +diag_086 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 SHORT SHORT INVALID +diag_087 0031002e0030 SHORT SHORT INVALID +diag_088 0031002e0035 SHORT SHORT INVALID +diag_089 003100650032 SHORT SHORT INVALID +diag_090 00310065002d0032 SHORT SHORT INVALID +diag_091 0031005f0030 SHORT SHORT INVALID +diag_092 002d0031005f0030 SHORT SHORT INVALID +diag_093 005f0031 SHORT SHORT INVALID +diag_094 0031005f SHORT SHORT INVALID +diag_095 0031005f005f0030 SHORT SHORT INVALID +diag_096 0031007f0030 SHORT SHORT INVALID +diag_097 0031004c SHORT SHORT INVALID +diag_098 0031006c SHORT SHORT INVALID +diag_099 00200031 SHORT SHORT INVALID +diag_100 00310020 SHORT SHORT INVALID +diag_101 00090031 SHORT SHORT INVALID +diag_102 0031000a SHORT SHORT INVALID +diag_103 0030007800310030 SHORT SHORT INVALID +diag_104 004e0061004e SHORT SHORT INVALID +diag_105 006e0061006e SHORT SHORT INVALID +diag_106 0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_107 002b0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_108 002d0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_109 0069006e0066 SHORT SHORT INVALID +diag_110 00310065003300300039 SHORT SHORT INVALID +diag_111 00310065002d0034003000300030 SHORT SHORT INVALID +diag_112 00310038003400340036003700340034003000370033003700300039003500350031003600310035 SHORT SHORT INVALID +diag_113 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 SHORT SHORT INVALID +diag_114 0000 SHORT SHORT INVALID +diag_115 00000031 SHORT SHORT INVALID +diag_116 00310000 SHORT SHORT INVALID +diag_117 003100000032 SHORT SHORT INVALID +diag_118 0661 SHORT SHORT INVALID +diag_119 00a00031 SHORT SHORT INVALID +diag_120 d800 SHORT SHORT INVALID +diag_121 0031d800 SHORT SHORT INVALID +diag_122 dc00 SHORT SHORT INVALID +diag_123 INT INT NULL VALID +diag_124 INT INT INVALID +diag_125 0030 INT INT 00000000 0 VALID +diag_126 002d0030 INT INT 00000000 0 VALID +diag_127 002b0030 INT INT 00000000 0 VALID +diag_128 00300030 INT INT 00000000 0 VALID +diag_129 0031 INT INT 00000001 1 VALID +diag_130 002b0031 INT INT 00000001 1 VALID +diag_131 002d0031 INT INT ffffffff -1 VALID +diag_132 003100320037 INT INT 0000007f 127 VALID +diag_133 003100320038 INT INT 00000080 128 VALID +diag_134 002d003100320038 INT INT ffffff80 -128 VALID +diag_135 002d003100320039 INT INT ffffff7f -129 VALID +diag_136 00330032003700360037 INT INT 00007fff 32767 VALID +diag_137 00330032003700360038 INT INT 00008000 32768 VALID +diag_138 002d00330032003700360038 INT INT ffff8000 -32768 VALID +diag_139 002d00330032003700360039 INT INT ffff7fff -32769 VALID +diag_140 0032003100340037003400380033003600340037 INT INT 7fffffff 2147483647 VALID +diag_141 0032003100340037003400380033003600340038 INT INT INVALID +diag_142 002d0032003100340037003400380033003600340038 INT INT 80000000 NULL VALID +diag_143 002d0032003100340037003400380033003600340039 INT INT INVALID +diag_144 0039003200320033003300370032003000330036003800350034003700370035003800300037 INT INT INVALID +diag_145 0039003200320033003300370032003000330036003800350034003700370035003800300038 INT INT INVALID +diag_146 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 INT INT INVALID +diag_147 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 INT INT INVALID +diag_148 0031002e0030 INT INT INVALID +diag_149 0031002e0035 INT INT INVALID +diag_150 003100650032 INT INT INVALID +diag_151 00310065002d0032 INT INT INVALID +diag_152 0031005f0030 INT INT 0000000a 10 VALID +diag_153 002d0031005f0030 INT INT fffffff6 -10 VALID +diag_154 005f0031 INT INT INVALID +diag_155 0031005f INT INT INVALID +diag_156 0031005f005f0030 INT INT INVALID +diag_157 0031007f0030 INT INT INVALID +diag_158 0031004c INT INT INVALID +diag_159 0031006c INT INT INVALID +diag_160 00200031 INT INT INVALID +diag_161 00310020 INT INT INVALID +diag_162 00090031 INT INT INVALID +diag_163 0031000a INT INT INVALID +diag_164 0030007800310030 INT INT INVALID +diag_165 004e0061004e INT INT INVALID +diag_166 006e0061006e INT INT INVALID +diag_167 0049006e00660069006e006900740079 INT INT INVALID +diag_168 002b0049006e00660069006e006900740079 INT INT INVALID +diag_169 002d0049006e00660069006e006900740079 INT INT INVALID +diag_170 0069006e0066 INT INT INVALID +diag_171 00310065003300300039 INT INT INVALID +diag_172 00310065002d0034003000300030 INT INT INVALID +diag_173 00310038003400340036003700340034003000370033003700300039003500350031003600310035 INT INT INVALID +diag_174 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 INT INT INVALID +diag_175 0000 INT INT INVALID +diag_176 00000031 INT INT INVALID +diag_177 00310000 INT INT INVALID +diag_178 003100000032 INT INT INVALID +diag_179 0661 INT INT INVALID +diag_180 00a00031 INT INT INVALID +diag_181 d800 INT INT INVALID +diag_182 0031d800 INT INT INVALID +diag_183 dc00 INT INT INVALID +diag_184 LONG LONG NULL VALID +diag_185 LONG LONG INVALID +diag_186 0030 LONG LONG 0000000000000000 0 VALID +diag_187 002d0030 LONG LONG 0000000000000000 0 VALID +diag_188 002b0030 LONG LONG INVALID +diag_189 00300030 LONG LONG 0000000000000000 0 VALID +diag_190 0031 LONG LONG 0000000000000001 1 VALID +diag_191 002b0031 LONG LONG INVALID +diag_192 002d0031 LONG LONG ffffffffffffffff -1 VALID +diag_193 003100320037 LONG LONG 000000000000007f 127 VALID +diag_194 003100320038 LONG LONG 0000000000000080 128 VALID +diag_195 002d003100320038 LONG LONG ffffffffffffff80 -128 VALID +diag_196 002d003100320039 LONG LONG ffffffffffffff7f -129 VALID +diag_197 00330032003700360037 LONG LONG 0000000000007fff 32767 VALID +diag_198 00330032003700360038 LONG LONG 0000000000008000 32768 VALID +diag_199 002d00330032003700360038 LONG LONG ffffffffffff8000 -32768 VALID +diag_200 002d00330032003700360039 LONG LONG ffffffffffff7fff -32769 VALID +diag_201 0032003100340037003400380033003600340037 LONG LONG 000000007fffffff 2147483647 VALID +diag_202 0032003100340037003400380033003600340038 LONG LONG 0000000080000000 2147483648 VALID +diag_203 002d0032003100340037003400380033003600340038 LONG LONG ffffffff80000000 -2147483648 VALID +diag_204 002d0032003100340037003400380033003600340039 LONG LONG ffffffff7fffffff -2147483649 VALID +diag_205 0039003200320033003300370032003000330036003800350034003700370035003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +diag_206 0039003200320033003300370032003000330036003800350034003700370035003800300038 LONG LONG INVALID +diag_207 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 LONG LONG 8000000000000000 NULL VALID +diag_208 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 LONG LONG INVALID +diag_209 0031002e0030 LONG LONG INVALID +diag_210 0031002e0035 LONG LONG INVALID +diag_211 003100650032 LONG LONG INVALID +diag_212 00310065002d0032 LONG LONG INVALID +diag_213 0031005f0030 LONG LONG 000000000000000a 10 VALID +diag_214 002d0031005f0030 LONG LONG fffffffffffffff6 -10 VALID +diag_215 005f0031 LONG LONG INVALID +diag_216 0031005f LONG LONG INVALID +diag_217 0031005f005f0030 LONG LONG INVALID +diag_218 0031007f0030 LONG LONG 000000000000000a 10 VALID +diag_219 0031004c LONG LONG 0000000000000001 1 VALID +diag_220 0031006c LONG LONG 0000000000000001 1 VALID +diag_221 00200031 LONG LONG INVALID +diag_222 00310020 LONG LONG INVALID +diag_223 00090031 LONG LONG INVALID +diag_224 0031000a LONG LONG INVALID +diag_225 0030007800310030 LONG LONG INVALID +diag_226 004e0061004e LONG LONG INVALID +diag_227 006e0061006e LONG LONG INVALID +diag_228 0049006e00660069006e006900740079 LONG LONG INVALID +diag_229 002b0049006e00660069006e006900740079 LONG LONG INVALID +diag_230 002d0049006e00660069006e006900740079 LONG LONG INVALID +diag_231 0069006e0066 LONG LONG INVALID +diag_232 00310065003300300039 LONG LONG INVALID +diag_233 00310065002d0034003000300030 LONG LONG INVALID +diag_234 00310038003400340036003700340034003000370033003700300039003500350031003600310035 LONG LONG INVALID +diag_235 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 LONG LONG INVALID +diag_236 0000 LONG LONG INVALID +diag_237 00000031 LONG LONG INVALID +diag_238 00310000 LONG LONG INVALID +diag_239 003100000032 LONG LONG INVALID +diag_240 0661 LONG LONG INVALID +diag_241 00a00031 LONG LONG INVALID +diag_242 d800 LONG LONG INVALID +diag_243 0031d800 LONG LONG INVALID +diag_244 dc00 LONG LONG INVALID +diag_245 FLOAT FLOAT NULL VALID +diag_246 FLOAT FLOAT INVALID +diag_247 0030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_248 002d0030 FLOAT FLOAT 80000000 BITS:80000000 VALID +diag_249 002b0030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_250 00300030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_251 0031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_252 002b0031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_253 002d0031 FLOAT FLOAT bf800000 BITS:bf800000 VALID +diag_254 003100320037 FLOAT FLOAT 42fe0000 BITS:42fe0000 VALID +diag_255 003100320038 FLOAT FLOAT 43000000 BITS:43000000 VALID +diag_256 002d003100320038 FLOAT FLOAT c3000000 BITS:c3000000 VALID +diag_257 002d003100320039 FLOAT FLOAT c3010000 BITS:c3010000 VALID +diag_258 00330032003700360037 FLOAT FLOAT 46fffe00 BITS:46fffe00 VALID +diag_259 00330032003700360038 FLOAT FLOAT 47000000 BITS:47000000 VALID +diag_260 002d00330032003700360038 FLOAT FLOAT c7000000 BITS:c7000000 VALID +diag_261 002d00330032003700360039 FLOAT FLOAT c7000100 BITS:c7000100 VALID +diag_262 0032003100340037003400380033003600340037 FLOAT FLOAT 4f000000 BITS:4f000000 VALID +diag_263 0032003100340037003400380033003600340038 FLOAT FLOAT 4f000000 BITS:4f000000 VALID +diag_264 002d0032003100340037003400380033003600340038 FLOAT FLOAT cf000000 BITS:cf000000 VALID +diag_265 002d0032003100340037003400380033003600340039 FLOAT FLOAT cf000000 BITS:cf000000 VALID +diag_266 0039003200320033003300370032003000330036003800350034003700370035003800300037 FLOAT FLOAT 5f000000 BITS:5f000000 VALID +diag_267 0039003200320033003300370032003000330036003800350034003700370035003800300038 FLOAT FLOAT 5f000000 BITS:5f000000 VALID +diag_268 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 FLOAT FLOAT df000000 BITS:df000000 VALID +diag_269 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 FLOAT FLOAT df000000 BITS:df000000 VALID +diag_270 0031002e0030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_271 0031002e0035 FLOAT FLOAT 3fc00000 BITS:3fc00000 VALID +diag_272 003100650032 FLOAT FLOAT 42c80000 BITS:42c80000 VALID +diag_273 00310065002d0032 FLOAT FLOAT 3c23d70a BITS:3c23d70a VALID +diag_274 0031005f0030 FLOAT FLOAT INVALID +diag_275 002d0031005f0030 FLOAT FLOAT INVALID +diag_276 005f0031 FLOAT FLOAT INVALID +diag_277 0031005f FLOAT FLOAT INVALID +diag_278 0031005f005f0030 FLOAT FLOAT INVALID +diag_279 0031007f0030 FLOAT FLOAT INVALID +diag_280 0031004c FLOAT FLOAT INVALID +diag_281 0031006c FLOAT FLOAT INVALID +diag_282 00200031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_283 00310020 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_284 00090031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_285 0031000a FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_286 0030007800310030 FLOAT FLOAT INVALID +diag_287 004e0061004e FLOAT FLOAT 7fc00000 NULL VALID +diag_288 006e0061006e FLOAT FLOAT INVALID +diag_289 0049006e00660069006e006900740079 FLOAT FLOAT 7f800000 BITS:7f800000 VALID +diag_290 002b0049006e00660069006e006900740079 FLOAT FLOAT 7f800000 BITS:7f800000 VALID +diag_291 002d0049006e00660069006e006900740079 FLOAT FLOAT ff800000 BITS:ff800000 VALID +diag_292 0069006e0066 FLOAT FLOAT INVALID +diag_293 00310065003300300039 FLOAT FLOAT INVALID +diag_294 00310065002d0034003000300030 FLOAT FLOAT INVALID +diag_295 00310038003400340036003700340034003000370033003700300039003500350031003600310035 FLOAT FLOAT 5f800000 BITS:5f800000 VALID +diag_296 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 FLOAT FLOAT INVALID +diag_297 0000 FLOAT FLOAT INVALID +diag_298 00000031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_299 00310000 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_300 003100000032 FLOAT FLOAT INVALID +diag_301 0661 FLOAT FLOAT INVALID +diag_302 00a00031 FLOAT FLOAT INVALID +diag_303 d800 FLOAT FLOAT INVALID +diag_304 0031d800 FLOAT FLOAT INVALID +diag_305 dc00 FLOAT FLOAT INVALID +diag_306 DOUBLE DOUBLE NULL VALID +diag_307 DOUBLE DOUBLE INVALID +diag_308 0030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_309 002d0030 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 VALID +diag_310 002b0030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_311 00300030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_312 0031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_313 002b0031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_314 002d0031 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 VALID +diag_315 003100320037 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 VALID +diag_316 003100320038 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 VALID +diag_317 002d003100320038 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 VALID +diag_318 002d003100320039 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 VALID +diag_319 00330032003700360037 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 VALID +diag_320 00330032003700360038 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 VALID +diag_321 002d00330032003700360038 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 VALID +diag_322 002d00330032003700360039 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 VALID +diag_323 0032003100340037003400380033003600340037 DOUBLE DOUBLE 41dfffffffc00000 BITS:41dfffffffc00000 VALID +diag_324 0032003100340037003400380033003600340038 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 VALID +diag_325 002d0032003100340037003400380033003600340038 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 VALID +diag_326 002d0032003100340037003400380033003600340039 DOUBLE DOUBLE c1e0000000200000 BITS:c1e0000000200000 VALID +diag_327 0039003200320033003300370032003000330036003800350034003700370035003800300037 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 VALID +diag_328 0039003200320033003300370032003000330036003800350034003700370035003800300038 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 VALID +diag_329 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 VALID +diag_330 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 VALID +diag_331 0031002e0030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_332 0031002e0035 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 VALID +diag_333 003100650032 DOUBLE DOUBLE 4059000000000000 BITS:4059000000000000 VALID +diag_334 00310065002d0032 DOUBLE DOUBLE 3f847ae147ae147b BITS:3f847ae147ae147b VALID +diag_335 0031005f0030 DOUBLE DOUBLE INVALID +diag_336 002d0031005f0030 DOUBLE DOUBLE INVALID +diag_337 005f0031 DOUBLE DOUBLE INVALID +diag_338 0031005f DOUBLE DOUBLE INVALID +diag_339 0031005f005f0030 DOUBLE DOUBLE INVALID +diag_340 0031007f0030 DOUBLE DOUBLE INVALID +diag_341 0031004c DOUBLE DOUBLE INVALID +diag_342 0031006c DOUBLE DOUBLE INVALID +diag_343 00200031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_344 00310020 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_345 00090031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_346 0031000a DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_347 0030007800310030 DOUBLE DOUBLE INVALID +diag_348 004e0061004e DOUBLE DOUBLE 7ff8000000000000 NULL VALID +diag_349 006e0061006e DOUBLE DOUBLE INVALID +diag_350 0049006e00660069006e006900740079 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 VALID +diag_351 002b0049006e00660069006e006900740079 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 VALID +diag_352 002d0049006e00660069006e006900740079 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 VALID +diag_353 0069006e0066 DOUBLE DOUBLE INVALID +diag_354 00310065003300300039 DOUBLE DOUBLE INVALID +diag_355 00310065002d0034003000300030 DOUBLE DOUBLE INVALID +diag_356 00310038003400340036003700340034003000370033003700300039003500350031003600310035 DOUBLE DOUBLE 43f0000000000000 BITS:43f0000000000000 VALID +diag_357 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 DOUBLE DOUBLE 4a511b0ec57e649a BITS:4a511b0ec57e649a VALID +diag_358 0000 DOUBLE DOUBLE INVALID +diag_359 00000031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_360 00310000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_361 003100000032 DOUBLE DOUBLE INVALID +diag_362 0661 DOUBLE DOUBLE INVALID +diag_363 00a00031 DOUBLE DOUBLE INVALID +diag_364 d800 DOUBLE DOUBLE INVALID +diag_365 0031d800 DOUBLE DOUBLE INVALID +diag_366 dc00 DOUBLE DOUBLE INVALID +wrap_pos_21e18 00320031003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_neg_21e18 002d00320031003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_25e18 00320035003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_42e18 00340032003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_63e18 00360033003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +overflow_u64_max 00310038003400340036003700340034003000370033003700300039003500350031003600310035 LONG LONG INVALID +overflow_u64 00310038003400340036003700340034003000370033003700300039003500350031003600310036 LONG LONG INVALID +float_hex_one 00300078003100700030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_f 003000780031007000300066 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_fraction_one 003000780031002e003000700030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_decimal_one_f 00310066 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_decimal_one_d 00310064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_hex_one 00300078003100700030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_f 003000780031007000300066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_fraction_one 003000780031002e003000700030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_decimal_one_f 00310066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_decimal_one_d 00310064 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +long_negative_suffix 002d0031004c LONG LONG ffffffffffffffff -1 VALID +long_separator_suffix 0031005f0030004c LONG LONG 000000000000000a 10 VALID +long_separator_before_suffix 0031005f004c LONG LONG INVALID +long_leading_zero_max 0030003000300039003200320033003300370032003000330036003800350034003700370035003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_max_suffix 0039003200320033003300370032003000330036003800350034003700370035003800300037006c LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_max_plus_one_suffix 0039003200320033003300370032003000330036003800350034003700370035003800300038004c LONG LONG INVALID +long_grouped_max 0039005f003200320033005f003300370032005f003000330036005f003800350034005f003700370035005f003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_del_grouped 0039007f003200320033007f003300370032007f003000330036007f003800350034007f003700370035007f003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +float_hex_one_d 003000780031007000300064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_upper_f 003000780031007000300046 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_upper_d 003000780031007000300044 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_upper_xp 00300058003100500030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_hex_one_d 003000780031007000300064 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_upper_f 003000780031007000300046 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_upper_d 003000780031007000300044 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_upper_xp 00300058003100500030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +float_min_subnormal 0031002e00340065002d00340035 FLOAT FLOAT 00000001 BITS:00000001 VALID +float_half_subnormal_underflow 00370065002d00340036 FLOAT FLOAT INVALID +float_max_finite 0033002e0034003000320038003200330035006500330038 FLOAT FLOAT 7f7fffff BITS:7f7fffff VALID +float_above_max_finite 0033002e0034003000320038003200330036006500330038 FLOAT FLOAT INVALID +double_min_subnormal 0034002e00390065002d003300320034 DOUBLE DOUBLE 0000000000000001 BITS:0000000000000001 VALID +double_half_subnormal_underflow 0032002e00340065002d003300320034 DOUBLE DOUBLE INVALID +double_max_finite 0031002e00370039003700360039003300310033003400380036003200330031003500370065003300300038 DOUBLE DOUBLE 7fefffffffffffff BITS:7fefffffffffffff VALID +double_above_max_finite 0031002e00370039003700360039003300310033003400380036003200330031003500390065003300300038 DOUBLE DOUBLE INVALID +float_cross_suffix_trailing_space 003100640020 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_cross_suffix_internal_space 003100200064 FLOAT FLOAT INVALID +float_mixed_suffix_chain 003100660064 FLOAT FLOAT INVALID +float_nan_cross_suffix 004e0061004e0064 FLOAT FLOAT INVALID +float_infinity_cross_suffix 0049006e00660069006e0069007400790064 FLOAT FLOAT INVALID +float_negative_cross_suffix 002d00310064 FLOAT FLOAT bf800000 BITS:bf800000 VALID +float_decimal_point_cross_suffix 0031002e0064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_missing_digits_cross_suffix 002e0064 FLOAT FLOAT INVALID +float_exponent_cross_suffix 0031006500300064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_leading_nul_cross_suffix 000000310064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_trailing_nul_cross_suffix 003100640000 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_cross_suffix_trailing_space 003100660020 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_cross_suffix_internal_space 003100200066 DOUBLE DOUBLE INVALID +double_mixed_suffix_chain 003100640066 DOUBLE DOUBLE INVALID +double_nan_cross_suffix 004e0061004e0066 DOUBLE DOUBLE INVALID +double_infinity_cross_suffix 0049006e00660069006e0069007400790066 DOUBLE DOUBLE INVALID +double_negative_cross_suffix 002d00310066 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 VALID +double_decimal_point_cross_suffix 0031002e0066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_missing_digits_cross_suffix 002e0066 DOUBLE DOUBLE INVALID +double_exponent_cross_suffix 0031006500300066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_leading_nul_cross_suffix 000000310066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_trailing_nul_cross_suffix 003100660000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv new file mode 100644 index 000000000..f48385efc --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv @@ -0,0 +1,8 @@ +case_id target input_kind input feature +runtime_two_digit_year TIMESTAMP TEXT 24-01-02 03:04:05.006Z reference_century +runtime_greedy_negative_year TIMESTAMP TEXT -1-01-01 00:00:00.000Z greedy_year_reference +runtime_zone_gap TIMESTAMP TEXT 2024-03-31T02:30:00Europe/Berlin tzdb_gap +runtime_zone_overlap TIMESTAMP TEXT 2024-10-27T02:30:00europe/berlin tzdb_overlap_case +runtime_zone_alias TIMESTAMP TEXT 2024-01-02T03:04:05EST zone_alias +runtime_historic_zone TIMESTAMP TEXT 1880-01-01T00:00:00Europe/Berlin tzdb_history +runtime_wrapped_zone TIMESTAMP TEXT 3509450-07-01 00:00:00.000Europe/Berlin parsed_year_rules diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv new file mode 100644 index 000000000..2cf2dcf70 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv @@ -0,0 +1,111 @@ +case_id target input_kind input outcome expected_raw feature +micro_pg_millis_z TIMESTAMP TEXT 2024-01-02 03:04:05.006Z VALID 1704164645006000 format_pg +micro_fraction_1 TIMESTAMP TEXT 2024-01-02T03:04:05.1Z VALID 1704164645100000 fraction_1 +micro_fraction_2 TIMESTAMP TEXT 2024-01-02T03:04:05.12Z VALID 1704164645120000 fraction_2 +micro_fraction_3 TIMESTAMP TEXT 2024-01-02T03:04:05.123Z VALID 1704164645123000 fraction_3 +micro_fraction_4 TIMESTAMP TEXT 2024-01-02T03:04:05.1234Z VALID 1704164645234000 fraction_4_compiled_behavior +micro_fraction_6 TIMESTAMP TEXT 2024-01-02T03:04:05.123456Z VALID 1704164645123456 fraction_6 +micro_seconds_z TIMESTAMP TEXT 2024-01-02T03:04:05Z VALID 1704164645000000 format_seconds +micro_hour TIMESTAMP TEXT 2024-01-02T03 VALID 1704164400000000 format_hour +micro_day TIMESTAMP TEXT 2024-01-02 VALID 1704153600000000 format_day +micro_week TIMESTAMP TEXT 2024-W01 VALID 1704067200000000 format_week +micro_month TIMESTAMP TEXT 2024-01 VALID 1704067200000000 format_month +micro_year TIMESTAMP TEXT 2024 VALID 1704067200000000 format_year +micro_negative_year TIMESTAMP TEXT -0001-01-01 VALID -62198755200000000 signed_year +micro_hour24 TIMESTAMP TEXT 2024-01-02T24:01:00Z VALID 1704153660000000 hour24_modulo +micro_numeric_zone TIMESTAMP TEXT 2024-01-02T03:04:05+01:30 VALID 1704159245000000 numeric_zone +micro_utc_offset TIMESTAMP TEXT 2024-01-02T03:04:05UTC+01:30 VALID 1704159245000000 utc_offset +micro_epoch_negative TIMESTAMP TEXT 1969-12-31T23:59:59.999999Z VALID -1 negative +micro_epoch_zero TIMESTAMP TEXT 1970-01-01T00:00:00Z VALID 0 zero +micro_parsed_long_min TIMESTAMP TEXT -292278-01-01 00:00:00.000Z VALID -9223372036854775808 parsed_sentinel_present_sql_null +micro_null TIMESTAMP TEXT NULL null +micro_bad_day TIMESTAMP TEXT 2024-02-30 INVALID calendar +micro_bad_week TIMESTAMP TEXT 2024-W54 INVALID week +micro_bad_fraction_empty TIMESTAMP TEXT 2024-01-02T03:04:05.Z INVALID fraction +micro_bad_fraction_5 TIMESTAMP TEXT 2024-01-02T03:04:05.12345Z INVALID fraction +micro_bad_fraction_7 TIMESTAMP TEXT 2024-01-02T03:04:05.1234567Z INVALID fraction +micro_bad_plus_year TIMESTAMP TEXT +2024-01-01 INVALID signed_year +micro_signed_hour TIMESTAMP TEXT 2024-01-01T-0:00:00Z VALID 1704067200000000 signed_fixed_hour +micro_signed_minute_second TIMESTAMP TEXT 2024-01-01T00:-0:-0Z VALID 1704067200000000 signed_fixed_fields +micro_signed_month TIMESTAMP TEXT 2024-+1-01 VALID 1704067200000000 signed_fixed_month +micro_negative_fraction TIMESTAMP TEXT 2024-01-01T00:00:00.-01Z VALID 1704067199999000 signed_fraction +micro_split_negative_fraction TIMESTAMP TEXT 2024-01-01T00:00:00.123-01Z VALID 1704067200122999 signed_micro_fraction +micro_nested_negative_year TIMESTAMP TEXT --001-01-01T00:00:00Z VALID -62135596800000000 nested_fixed_year +micro_negative_week TIMESTAMP TEXT 2024-W-1 VALID 1704067200000000 signed_week +micro_int_min_year TIMESTAMP TEXT -2147483648-01-01 00:00:00.000Z VALID -9223372036854775808 year_micros_saturation +micro_malformed_high TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,d800,0031 INVALID utf16_replacement +micro_malformed_low TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,dc00,0031 INVALID utf16_replacement +micro_empty TIMESTAMP TEXT INVALID empty +micro_trailing_junk TIMESTAMP TEXT 2024-01-01T00:00:00Zjunk INVALID trailing_junk +micro_nonascii_zone_lookalike TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,0030,0031,002d,0030,0031,0054,0030,0030,003a,0030,0030,003a,0030,0030,0045,0075,0072,006f,0070,0065,002f,0042,0065,0072,006c,0131,006e INVALID unicode_zone_lookalike +micro_fixed_year_plus TIMESTAMP TEXT +2024 INVALID fixed_year_sign +micro_fixed_year_underscore TIMESTAMP TEXT 2024_01_01 INVALID separator +micro_reachable_max TIMESTAMP TEXT 44135800-09-20 06:17:51.147Z VALID 9223372036854775800 micro_wrapped_lattice_max +micro_reachable_max_successor_wrap TIMESTAMP TEXT 44135800-09-20 06:17:51.148Z VALID -9223372036854774816 micro_unchecked_wrap +micro_nonwrapped_near_max TIMESTAMP TEXT 294247-01-10 04:00:54.775Z VALID 9223372036854775000 micro_near_max +micro_saturation_plus_millis TIMESTAMP TEXT -292278-01-01 00:00:00.001Z VALID -9223372036854774808 year_saturation_plus +micro_reachable_min_plus_eight TIMESTAMP TEXT 28937395-06-09 13:30:22.805Z VALID -9223372036854775800 micro_wrapped_lattice_min +nano_pg_millis_z TIMESTAMP_NANOS TEXT 2024-01-02 03:04:05.006Z VALID 1704164645006000000 format_pg +nano_fraction_1 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1Z VALID 1704164645100000000 fraction_1 +nano_fraction_2 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.12Z VALID 1704164645120000000 fraction_2 +nano_fraction_3 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.123Z VALID 1704164645123000000 fraction_3 +nano_fraction_4 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1234Z VALID 1704164645234000000 fraction_4_compiled_behavior +nano_fraction_6 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.123456Z VALID 1704164645123456000 fraction_6 +nano_seconds_z TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05Z VALID 1704164645000000000 format_seconds +nano_hour TIMESTAMP_NANOS TEXT 2024-01-02T03 VALID 1704164400000000000 format_hour +nano_day TIMESTAMP_NANOS TEXT 2024-01-02 VALID 1704153600000000000 format_day +nano_week TIMESTAMP_NANOS TEXT 2024-W01 VALID 1704067200000000000 format_week +nano_month TIMESTAMP_NANOS TEXT 2024-01 VALID 1704067200000000000 format_month +nano_year TIMESTAMP_NANOS TEXT 2024 VALID 1704067200000000000 format_year +nano_negative_year TIMESTAMP_NANOS TEXT -0001-01-01 INVALID nano_overflow_after_parse +nano_hour24 TIMESTAMP_NANOS TEXT 2024-01-02T24:01:00Z VALID 1704153660000000000 hour24_modulo +nano_numeric_zone TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05+01:30 VALID 1704159245000000000 numeric_zone +nano_utc_offset TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05UTC+01:30 VALID 1704159245000000000 utc_offset +nano_epoch_negative TIMESTAMP_NANOS TEXT 1969-12-31T23:59:59.999999Z VALID -1000 negative +nano_epoch_zero TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00Z VALID 0 zero +nano_parsed_long_min TIMESTAMP_NANOS TEXT -292278-01-01 00:00:00.000Z INVALID nano_overflow_after_parse +nano_null TIMESTAMP_NANOS TEXT NULL null +nano_bad_day TIMESTAMP_NANOS TEXT 2024-02-30 INVALID calendar +nano_bad_week TIMESTAMP_NANOS TEXT 2024-W54 INVALID week +nano_bad_fraction_empty TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.Z INVALID fraction +nano_bad_fraction_5 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.12345Z INVALID fraction +nano_bad_fraction_7 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1234567Z INVALID fraction +nano_bad_plus_year TIMESTAMP_NANOS TEXT +2024-01-01 INVALID signed_year +nano_signed_hour TIMESTAMP_NANOS TEXT 2024-01-01T-0:00:00Z VALID 1704067200000000000 signed_fixed_hour +nano_signed_minute_second TIMESTAMP_NANOS TEXT 2024-01-01T00:-0:-0Z VALID 1704067200000000000 signed_fixed_fields +nano_signed_month TIMESTAMP_NANOS TEXT 2024-+1-01 VALID 1704067200000000000 signed_fixed_month +nano_negative_fraction TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00.-01Z VALID 1704067199999000000 signed_fraction +nano_split_negative_fraction TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00.123-01Z VALID 1704067200122999000 signed_micro_fraction +nano_nested_negative_year TIMESTAMP_NANOS TEXT --001-01-01T00:00:00Z INVALID nano_overflow_after_parse +nano_negative_week TIMESTAMP_NANOS TEXT 2024-W-1 VALID 1704067200000000000 signed_week +nano_int_min_year TIMESTAMP_NANOS TEXT -2147483648-01-01 00:00:00.000Z INVALID nano_overflow_after_parse +nano_malformed_high TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,d800,0031 INVALID utf16_replacement +nano_malformed_low TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,dc00,0031 INVALID utf16_replacement +nano_empty TIMESTAMP_NANOS TEXT INVALID empty +nano_trailing_junk TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00Zjunk INVALID trailing_junk +nano_nonascii_zone_lookalike TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,0030,0031,002d,0030,0031,0054,0030,0030,003a,0030,0030,003a,0030,0030,0045,0075,0072,006f,0070,0065,002f,0042,0065,0072,006c,0131,006e INVALID unicode_zone_lookalike +nano_fixed_year_plus TIMESTAMP_NANOS TEXT +2024 INVALID fixed_year_sign +nano_fixed_year_underscore TIMESTAMP_NANOS TEXT 2024_01_01 INVALID separator +nano_reachable_max TIMESTAMP_NANOS TEXT 44135800-09-20 06:17:51.147Z INVALID nano_overflow_after_parse +nano_reachable_max_successor_wrap TIMESTAMP_NANOS TEXT 44135800-09-20 06:17:51.148Z INVALID nano_overflow_after_parse +nano_nonwrapped_near_max TIMESTAMP_NANOS TEXT 294247-01-10 04:00:54.775Z INVALID nano_overflow_after_parse +nano_saturation_plus_millis TIMESTAMP_NANOS TEXT -292278-01-01 00:00:00.001Z INVALID nano_overflow_after_parse +nano_reachable_min_plus_eight TIMESTAMP_NANOS TEXT 28937395-06-09 13:30:22.805Z INVALID nano_overflow_after_parse +nano_upper_bound TIMESTAMP_NANOS TEXT 2262-04-11T23:47:16.854775Z VALID 9223372036854775000 nano_upper_bound +nano_upper_bound_plus_one TIMESTAMP_NANOS TEXT 2262-04-11T23:47:16.854776Z INVALID nano_overflow +nano_lower_bound TIMESTAMP_NANOS TEXT 1677-09-21T00:12:43.145225Z VALID -9223372036854775000 nano_lower_bound +nano_lower_bound_minus_one TIMESTAMP_NANOS TEXT 1677-09-21T00:12:43.145224Z INVALID nano_overflow +micro_fraction_fixed_plus TIMESTAMP TEXT 1970-01-01T00:00:00.+12Z VALID 12000 fixed_fraction_priority +micro_fraction_fixed_underscore TIMESTAMP TEXT 1970-01-01T00:00:00.1_2Z VALID 12000 fixed_fraction_priority +micro_fraction_fixed_numeric_zone TIMESTAMP TEXT 1970-01-01T00:00:00.10000 VALID 100000 fixed_fraction_priority +micro_fraction_extra_zone_tail TIMESTAMP TEXT 1970-01-01T00:00:00.10000Z INVALID fixed_fraction_priority +micro_fraction_split_signed TIMESTAMP TEXT 1970-01-01T00:00:00.1_2+34Z VALID 12034 fixed_fraction_priority +nano_fraction_fixed_plus TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.+12Z VALID 12000000 fixed_fraction_priority +nano_fraction_fixed_underscore TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.1_2Z VALID 12000000 fixed_fraction_priority +nano_fraction_fixed_numeric_zone TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.10000 VALID 100000000 fixed_fraction_priority +nano_fraction_extra_zone_tail TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.10000Z INVALID fixed_fraction_priority +nano_fraction_split_signed TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.1_2+34Z VALID 12034000 fixed_fraction_priority +micro_iso_week_retained_leap_1297 TIMESTAMP TEXT 1297-W01 VALID -21237724800000000 iso_week_source_leap +micro_iso_week_retained_leap_1697 TIMESTAMP TEXT 1697-W01 VALID -8614944000000000 iso_week_source_leap +nano_iso_week_retained_leap_1297 TIMESTAMP_NANOS TEXT 1297-W01 INVALID nano_overflow_after_parse +nano_iso_week_retained_leap_1697 TIMESTAMP_NANOS TEXT 1697-W01 VALID -8614944000000000000 iso_week_source_leap diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv new file mode 100644 index 000000000..f9f4decd0 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv @@ -0,0 +1,91 @@ +# case_id input_type target_type input_utf16_hex expected_wire_type expected_utf8_hex expected_sql_utf16_hex +string_string_ascii STRING STRING 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +string_string_lower STRING STRING 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +string_string_controls STRING STRING 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +string_string_empty STRING STRING VARCHAR +string_string_spaces STRING STRING 002000610020 VARCHAR 206120 002000610020 +string_string_nul STRING STRING 006100000062 VARCHAR 610062 006100000062 +string_string_bmp STRING STRING 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +string_string_pair STRING STRING d83dde00 VARCHAR f09f9880 d83dde00 +string_string_high STRING STRING d800 VARCHAR 3f 003f +string_string_low STRING STRING dc00 VARCHAR 3f 003f +string_string_high_ascii STRING STRING d8000041 VARCHAR 3f41 003f0041 +string_string_high_high_low STRING STRING d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +string_string_low_high_ascii STRING STRING dc00d8000041 VARCHAR 3f3f41 003f003f0041 +string_string_question STRING STRING 003f VARCHAR 3f 003f +string_string_null STRING STRING VARCHAR +string_varchar_ascii STRING VARCHAR 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +string_varchar_lower STRING VARCHAR 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +string_varchar_controls STRING VARCHAR 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +string_varchar_empty STRING VARCHAR VARCHAR +string_varchar_spaces STRING VARCHAR 002000610020 VARCHAR 206120 002000610020 +string_varchar_nul STRING VARCHAR 006100000062 VARCHAR 610062 006100000062 +string_varchar_bmp STRING VARCHAR 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +string_varchar_pair STRING VARCHAR d83dde00 VARCHAR f09f9880 d83dde00 +string_varchar_high STRING VARCHAR d800 VARCHAR 3f 003f +string_varchar_low STRING VARCHAR dc00 VARCHAR 3f 003f +string_varchar_high_ascii STRING VARCHAR d8000041 VARCHAR 3f41 003f0041 +string_varchar_high_high_low STRING VARCHAR d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +string_varchar_low_high_ascii STRING VARCHAR dc00d8000041 VARCHAR 3f3f41 003f003f0041 +string_varchar_question STRING VARCHAR 003f VARCHAR 3f 003f +string_varchar_null STRING VARCHAR VARCHAR +string_symbol_ascii STRING SYMBOL 00510075006500730074 SYMBOL 5175657374 00510075006500730074 +string_symbol_lower STRING SYMBOL 00710075006500730074 SYMBOL 7175657374 00710075006500730074 +string_symbol_controls STRING SYMBOL 00610009000d000a0062 SYMBOL 61090d0a62 00610009000d000a0062 +string_symbol_empty STRING SYMBOL SYMBOL +string_symbol_spaces STRING SYMBOL 002000610020 SYMBOL 206120 002000610020 +string_symbol_nul STRING SYMBOL 006100000062 SYMBOL 610062 006100000062 +string_symbol_bmp STRING SYMBOL 00e94e2d SYMBOL c3a9e4b8ad 00e94e2d +string_symbol_pair STRING SYMBOL d83dde00 SYMBOL f09f9880 d83dde00 +string_symbol_high STRING SYMBOL d800 SYMBOL 3f 003f +string_symbol_low STRING SYMBOL dc00 SYMBOL 3f 003f +string_symbol_high_ascii STRING SYMBOL d8000041 SYMBOL 3f41 003f0041 +string_symbol_high_high_low STRING SYMBOL d800d83dde00 SYMBOL 3ff09f9880 003fd83dde00 +string_symbol_low_high_ascii STRING SYMBOL dc00d8000041 SYMBOL 3f3f41 003f003f0041 +string_symbol_question STRING SYMBOL 003f SYMBOL 3f 003f +string_symbol_null STRING SYMBOL SYMBOL +symbol_string_ascii SYMBOL STRING 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +symbol_string_lower SYMBOL STRING 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +symbol_string_controls SYMBOL STRING 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +symbol_string_empty SYMBOL STRING VARCHAR +symbol_string_spaces SYMBOL STRING 002000610020 VARCHAR 206120 002000610020 +symbol_string_nul SYMBOL STRING 006100000062 VARCHAR 610062 006100000062 +symbol_string_bmp SYMBOL STRING 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +symbol_string_pair SYMBOL STRING d83dde00 VARCHAR f09f9880 d83dde00 +symbol_string_high SYMBOL STRING d800 VARCHAR 3f 003f +symbol_string_low SYMBOL STRING dc00 VARCHAR 3f 003f +symbol_string_high_ascii SYMBOL STRING d8000041 VARCHAR 3f41 003f0041 +symbol_string_high_high_low SYMBOL STRING d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +symbol_string_low_high_ascii SYMBOL STRING dc00d8000041 VARCHAR 3f3f41 003f003f0041 +symbol_string_question SYMBOL STRING 003f VARCHAR 3f 003f +symbol_string_null SYMBOL STRING VARCHAR +symbol_varchar_ascii SYMBOL VARCHAR 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +symbol_varchar_lower SYMBOL VARCHAR 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +symbol_varchar_controls SYMBOL VARCHAR 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +symbol_varchar_empty SYMBOL VARCHAR VARCHAR +symbol_varchar_spaces SYMBOL VARCHAR 002000610020 VARCHAR 206120 002000610020 +symbol_varchar_nul SYMBOL VARCHAR 006100000062 VARCHAR 610062 006100000062 +symbol_varchar_bmp SYMBOL VARCHAR 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +symbol_varchar_pair SYMBOL VARCHAR d83dde00 VARCHAR f09f9880 d83dde00 +symbol_varchar_high SYMBOL VARCHAR d800 VARCHAR 3f 003f +symbol_varchar_low SYMBOL VARCHAR dc00 VARCHAR 3f 003f +symbol_varchar_high_ascii SYMBOL VARCHAR d8000041 VARCHAR 3f41 003f0041 +symbol_varchar_high_high_low SYMBOL VARCHAR d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +symbol_varchar_low_high_ascii SYMBOL VARCHAR dc00d8000041 VARCHAR 3f3f41 003f003f0041 +symbol_varchar_question SYMBOL VARCHAR 003f VARCHAR 3f 003f +symbol_varchar_null SYMBOL VARCHAR VARCHAR +symbol_symbol_ascii SYMBOL SYMBOL 00510075006500730074 SYMBOL 5175657374 00510075006500730074 +symbol_symbol_lower SYMBOL SYMBOL 00710075006500730074 SYMBOL 7175657374 00710075006500730074 +symbol_symbol_controls SYMBOL SYMBOL 00610009000d000a0062 SYMBOL 61090d0a62 00610009000d000a0062 +symbol_symbol_empty SYMBOL SYMBOL SYMBOL +symbol_symbol_spaces SYMBOL SYMBOL 002000610020 SYMBOL 206120 002000610020 +symbol_symbol_nul SYMBOL SYMBOL 006100000062 SYMBOL 610062 006100000062 +symbol_symbol_bmp SYMBOL SYMBOL 00e94e2d SYMBOL c3a9e4b8ad 00e94e2d +symbol_symbol_pair SYMBOL SYMBOL d83dde00 SYMBOL f09f9880 d83dde00 +symbol_symbol_high SYMBOL SYMBOL d800 SYMBOL 3f 003f +symbol_symbol_low SYMBOL SYMBOL dc00 SYMBOL 3f 003f +symbol_symbol_high_ascii SYMBOL SYMBOL d8000041 SYMBOL 3f41 003f0041 +symbol_symbol_high_high_low SYMBOL SYMBOL d800d83dde00 SYMBOL 3ff09f9880 003fd83dde00 +symbol_symbol_low_high_ascii SYMBOL SYMBOL dc00d8000041 SYMBOL 3f3f41 003f003f0041 +symbol_symbol_question SYMBOL SYMBOL 003f SYMBOL 3f 003f +symbol_symbol_null SYMBOL SYMBOL SYMBOL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv new file mode 100644 index 000000000..050717e7d --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv @@ -0,0 +1,103 @@ +# case_id input_kind input source_unit target_type expected_wire_value expected_sql_value +millis_micro_zero LONG 0 MILLIS TIMESTAMP 0 0 +millis_micro_neg_one LONG -1 MILLIS TIMESTAMP -1000 -1000 +millis_micro_pos_one LONG 1 MILLIS TIMESTAMP 1000 1000 +millis_micro_low_valid LONG -9223372036854775 MILLIS TIMESTAMP -9223372036854775000 -9223372036854775000 +millis_micro_low_invalid LONG -9223372036854776 MILLIS TIMESTAMP +millis_micro_high_valid LONG 9223372036854775 MILLIS TIMESTAMP 9223372036854775000 9223372036854775000 +millis_micro_high_invalid LONG 9223372036854776 MILLIS TIMESTAMP +millis_micro_long_min LONG -9223372036854775808 MILLIS TIMESTAMP +millis_nano_zero LONG 0 MILLIS TIMESTAMP_NS 0 0 +millis_nano_neg_one LONG -1 MILLIS TIMESTAMP_NS -1000000 -1000000 +millis_nano_pos_one LONG 1 MILLIS TIMESTAMP_NS 1000000 1000000 +millis_nano_low_valid LONG -9223372036854 MILLIS TIMESTAMP_NS -9223372036854000000 -9223372036854000000 +millis_nano_low_invalid LONG -9223372036855 MILLIS TIMESTAMP_NS +millis_nano_high_valid LONG 9223372036854 MILLIS TIMESTAMP_NS 9223372036854000000 9223372036854000000 +millis_nano_high_invalid LONG 9223372036855 MILLIS TIMESTAMP_NS +millis_nano_long_min LONG -9223372036854775808 MILLIS TIMESTAMP_NS +seconds_micro_zero LONG 0 SECONDS TIMESTAMP 0 0 +seconds_micro_neg_one LONG -1 SECONDS TIMESTAMP -1000000 -1000000 +seconds_micro_pos_one LONG 1 SECONDS TIMESTAMP 1000000 1000000 +seconds_micro_low_valid LONG -9223372036854 SECONDS TIMESTAMP -9223372036854000000 -9223372036854000000 +seconds_micro_low_invalid LONG -9223372036855 SECONDS TIMESTAMP +seconds_micro_high_valid LONG 9223372036854 SECONDS TIMESTAMP 9223372036854000000 9223372036854000000 +seconds_micro_high_invalid LONG 9223372036855 SECONDS TIMESTAMP +seconds_micro_long_min LONG -9223372036854775808 SECONDS TIMESTAMP +seconds_nano_zero LONG 0 SECONDS TIMESTAMP_NS 0 0 +seconds_nano_neg_one LONG -1 SECONDS TIMESTAMP_NS -1000000000 -1000000000 +seconds_nano_pos_one LONG 1 SECONDS TIMESTAMP_NS 1000000000 1000000000 +seconds_nano_low_valid LONG -9223372036 SECONDS TIMESTAMP_NS -9223372036000000000 -9223372036000000000 +seconds_nano_low_invalid LONG -9223372037 SECONDS TIMESTAMP_NS +seconds_nano_high_valid LONG 9223372036 SECONDS TIMESTAMP_NS 9223372036000000000 9223372036000000000 +seconds_nano_high_invalid LONG 9223372037 SECONDS TIMESTAMP_NS +seconds_nano_long_min LONG -9223372036854775808 SECONDS TIMESTAMP_NS +minutes_micro_zero LONG 0 MINUTES TIMESTAMP 0 0 +minutes_micro_neg_one LONG -1 MINUTES TIMESTAMP -60000000 -60000000 +minutes_micro_pos_one LONG 1 MINUTES TIMESTAMP 60000000 60000000 +minutes_micro_low_valid LONG -153722867280 MINUTES TIMESTAMP -9223372036800000000 -9223372036800000000 +minutes_micro_low_invalid LONG -153722867281 MINUTES TIMESTAMP +minutes_micro_high_valid LONG 153722867280 MINUTES TIMESTAMP 9223372036800000000 9223372036800000000 +minutes_micro_high_invalid LONG 153722867281 MINUTES TIMESTAMP +minutes_micro_long_min LONG -9223372036854775808 MINUTES TIMESTAMP +minutes_nano_zero LONG 0 MINUTES TIMESTAMP_NS 0 0 +minutes_nano_neg_one LONG -1 MINUTES TIMESTAMP_NS -60000000000 -60000000000 +minutes_nano_pos_one LONG 1 MINUTES TIMESTAMP_NS 60000000000 60000000000 +minutes_nano_low_valid LONG -153722867 MINUTES TIMESTAMP_NS -9223372020000000000 -9223372020000000000 +minutes_nano_low_invalid LONG -153722868 MINUTES TIMESTAMP_NS +minutes_nano_high_valid LONG 153722867 MINUTES TIMESTAMP_NS 9223372020000000000 9223372020000000000 +minutes_nano_high_invalid LONG 153722868 MINUTES TIMESTAMP_NS +minutes_nano_long_min LONG -9223372036854775808 MINUTES TIMESTAMP_NS +hours_micro_zero LONG 0 HOURS TIMESTAMP 0 0 +hours_micro_neg_one LONG -1 HOURS TIMESTAMP -3600000000 -3600000000 +hours_micro_pos_one LONG 1 HOURS TIMESTAMP 3600000000 3600000000 +hours_micro_low_valid LONG -2562047788 HOURS TIMESTAMP -9223372036800000000 -9223372036800000000 +hours_micro_low_invalid LONG -2562047789 HOURS TIMESTAMP +hours_micro_high_valid LONG 2562047788 HOURS TIMESTAMP 9223372036800000000 9223372036800000000 +hours_micro_high_invalid LONG 2562047789 HOURS TIMESTAMP +hours_micro_long_min LONG -9223372036854775808 HOURS TIMESTAMP +hours_nano_zero LONG 0 HOURS TIMESTAMP_NS 0 0 +hours_nano_neg_one LONG -1 HOURS TIMESTAMP_NS -3600000000000 -3600000000000 +hours_nano_pos_one LONG 1 HOURS TIMESTAMP_NS 3600000000000 3600000000000 +hours_nano_low_valid LONG -2562047 HOURS TIMESTAMP_NS -9223369200000000000 -9223369200000000000 +hours_nano_low_invalid LONG -2562048 HOURS TIMESTAMP_NS +hours_nano_high_valid LONG 2562047 HOURS TIMESTAMP_NS 9223369200000000000 9223369200000000000 +hours_nano_high_invalid LONG 2562048 HOURS TIMESTAMP_NS +hours_nano_long_min LONG -9223372036854775808 HOURS TIMESTAMP_NS +days_micro_zero LONG 0 DAYS TIMESTAMP 0 0 +days_micro_neg_one LONG -1 DAYS TIMESTAMP -86400000000 -86400000000 +days_micro_pos_one LONG 1 DAYS TIMESTAMP 86400000000 86400000000 +days_micro_low_valid LONG -106751991 DAYS TIMESTAMP -9223372022400000000 -9223372022400000000 +days_micro_low_invalid LONG -106751992 DAYS TIMESTAMP +days_micro_high_valid LONG 106751991 DAYS TIMESTAMP 9223372022400000000 9223372022400000000 +days_micro_high_invalid LONG 106751992 DAYS TIMESTAMP +days_micro_long_min LONG -9223372036854775808 DAYS TIMESTAMP +days_nano_zero LONG 0 DAYS TIMESTAMP_NS 0 0 +days_nano_neg_one LONG -1 DAYS TIMESTAMP_NS -86400000000000 -86400000000000 +days_nano_pos_one LONG 1 DAYS TIMESTAMP_NS 86400000000000 86400000000000 +days_nano_low_valid LONG -106751 DAYS TIMESTAMP_NS -9223286400000000000 -9223286400000000000 +days_nano_low_invalid LONG -106752 DAYS TIMESTAMP_NS +days_nano_high_valid LONG 106751 DAYS TIMESTAMP_NS 9223286400000000000 9223286400000000000 +days_nano_high_invalid LONG 106752 DAYS TIMESTAMP_NS +days_nano_long_min LONG -9223372036854775808 DAYS TIMESTAMP_NS +instant_micro_zero INSTANT 1970-01-01T00:00:00Z - TIMESTAMP 0 0 +instant_micro_pos_submicro INSTANT 1970-01-01T00:00:00.000000999Z - TIMESTAMP 0 0 +instant_micro_neg_submicro INSTANT 1969-12-31T23:59:59.999999999Z - TIMESTAMP -1 -1 +instant_micro_min INSTANT -290308-12-21T19:59:05.224192Z - TIMESTAMP -9223372036854775808 NULL +instant_micro_min_fraction INSTANT -290308-12-21T19:59:05.224192999Z - TIMESTAMP -9223372036854775808 NULL +instant_micro_min_plus_one INSTANT -290308-12-21T19:59:05.224193Z - TIMESTAMP -9223372036854775807 -9223372036854775807 +instant_micro_below_min INSTANT -290308-12-21T19:59:05.224191999Z - TIMESTAMP +instant_micro_max INSTANT +294247-01-10T04:00:54.775807Z - TIMESTAMP 9223372036854775807 9223372036854775807 +instant_micro_max_fraction INSTANT +294247-01-10T04:00:54.775807999Z - TIMESTAMP 9223372036854775807 9223372036854775807 +instant_micro_above_max INSTANT +294247-01-10T04:00:54.775808Z - TIMESTAMP +instant_nano_zero INSTANT 1970-01-01T00:00:00Z - TIMESTAMP_NS 0 0 +instant_nano_pos_submicro INSTANT 1970-01-01T00:00:00.000000999Z - TIMESTAMP_NS 999 999 +instant_nano_neg_submicro INSTANT 1969-12-31T23:59:59.999999999Z - TIMESTAMP_NS -1 -1 +instant_nano_min INSTANT 1677-09-21T00:12:43.145224192Z - TIMESTAMP_NS -9223372036854775808 NULL +instant_nano_min_plus_one INSTANT 1677-09-21T00:12:43.145224193Z - TIMESTAMP_NS -9223372036854775807 -9223372036854775807 +instant_nano_below_min INSTANT 1677-09-21T00:12:43.145224191Z - TIMESTAMP_NS +instant_nano_max INSTANT 2262-04-11T23:47:16.854775807Z - TIMESTAMP_NS 9223372036854775807 9223372036854775807 +instant_nano_above_max INSTANT 2262-04-11T23:47:16.854775808Z - TIMESTAMP_NS +instant_min_micro INSTANT -1000000000-01-01T00:00:00Z - TIMESTAMP +instant_max_micro INSTANT +1000000000-12-31T23:59:59.999999999Z - TIMESTAMP +instant_min_nano INSTANT -1000000000-01-01T00:00:00Z - TIMESTAMP_NS +instant_max_nano INSTANT +1000000000-12-31T23:59:59.999999999Z - TIMESTAMP_NS diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv new file mode 100644 index 000000000..69ceaa4bd --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv @@ -0,0 +1,64 @@ +# case_id input_kind unit value nanos outcome expected_text +micro_min LONG MICROS -9223372036854775808 VALUE +micro_min_plus_one LONG MICROS -9223372036854775807 VALUE -290308-01-01T19:59:05.224Z +micro_minus_1000 LONG MICROS -1000 VALUE 1969-12-31T23:59:59.999Z +micro_minus_999 LONG MICROS -999 VALUE 1969-12-31T23:59:59.999Z +micro_minus_one LONG MICROS -1 VALUE 1969-12-31T23:59:59.999Z +micro_zero LONG MICROS 0 VALUE 1970-01-01T00:00:00.000Z +micro_one LONG MICROS 1 VALUE 1970-01-01T00:00:00.000Z +micro_999 LONG MICROS 999 VALUE 1970-01-01T00:00:00.000Z +micro_1000 LONG MICROS 1000 VALUE 1970-01-01T00:00:00.001Z +micro_max LONG MICROS 9223372036854775807 VALUE 294247-01-10T04:00:54.775Z +micro_year_minus_one LONG MICROS -62198755200000000 VALUE 000-1-01-01T00:00:00.000Z +micro_year_zero LONG MICROS -62167219200000000 VALUE 0001-01-01T00:00:00.000Z +micro_year_one LONG MICROS -62135596800000000 VALUE 0001-01-01T00:00:00.000Z +micro_year_99 LONG MICROS -59042995200000000 VALUE 0099-01-01T00:00:00.000Z +micro_year_100 LONG MICROS -59011459200000000 VALUE 0100-01-01T00:00:00.000Z +micro_year_9999 LONG MICROS 253370764800000000 VALUE 9999-01-01T00:00:00.000Z +micro_year_10000 LONG MICROS 253402300800000000 VALUE 10000-01-01T00:00:00.000Z +nano_min LONG NANOS -9223372036854775808 VALUE 1677-09-21T00:12:43.145Z +nano_min_plus_one LONG NANOS -9223372036854775807 VALUE 1677-09-21T00:12:43.145Z +nano_minus_1000 LONG NANOS -1000 VALUE 1969-12-31T23:59:59.999Z +nano_minus_999 LONG NANOS -999 VALUE 1970-01-01T00:00:00.000Z +nano_minus_one LONG NANOS -1 VALUE 1970-01-01T00:00:00.000Z +nano_zero LONG NANOS 0 VALUE 1970-01-01T00:00:00.000Z +nano_one LONG NANOS 1 VALUE 1970-01-01T00:00:00.000Z +nano_999 LONG NANOS 999 VALUE 1970-01-01T00:00:00.000Z +nano_1000 LONG NANOS 1000 VALUE 1970-01-01T00:00:00.000Z +nano_max LONG NANOS 9223372036854775807 VALUE 2262-04-11T23:47:16.854Z +millis_negative LONG MILLIS -1 VALUE 1969-12-31T23:59:59.999Z +millis_positive LONG MILLIS 1 VALUE 1970-01-01T00:00:00.001Z +seconds_negative LONG SECONDS -1 VALUE 1969-12-31T23:59:59.000Z +seconds_positive LONG SECONDS 1 VALUE 1970-01-01T00:00:01.000Z +minutes_positive LONG MINUTES 1 VALUE 1970-01-01T00:01:00.000Z +hours_negative LONG HOURS -1 VALUE 1969-12-31T23:00:00.000Z +days_positive LONG DAYS 1 VALUE 1970-01-02T00:00:00.000Z +days_overflow LONG DAYS 9223372036854775807 INVALID +millis_max_accepted LONG MILLIS 9223372036854775 VALUE 294247-01-10T04:00:54.775Z +millis_positive_overflow LONG MILLIS 9223372036854776 INVALID +millis_min_accepted LONG MILLIS -9223372036854775 VALUE -290308-01-01T19:59:05.225Z +millis_negative_overflow LONG MILLIS -9223372036854776 INVALID +seconds_max_accepted LONG SECONDS 9223372036854 VALUE 294247-01-10T04:00:54.000Z +seconds_positive_overflow LONG SECONDS 9223372036855 INVALID +seconds_min_accepted LONG SECONDS -9223372036854 VALUE -290308-01-01T19:59:06.000Z +seconds_negative_overflow LONG SECONDS -9223372036855 INVALID +minutes_max_accepted LONG MINUTES 153722867280 VALUE 294247-01-10T04:00:00.000Z +minutes_positive_overflow LONG MINUTES 153722867281 INVALID +minutes_min_accepted LONG MINUTES -153722867280 VALUE -290308-01-01T20:00:00.000Z +minutes_negative_overflow LONG MINUTES -153722867281 INVALID +hours_max_accepted LONG HOURS 2562047788 VALUE 294247-01-10T04:00:00.000Z +hours_positive_overflow LONG HOURS 2562047789 INVALID +hours_min_accepted LONG HOURS -2562047788 VALUE -290308-01-01T20:00:00.000Z +hours_negative_overflow LONG HOURS -2562047789 INVALID +days_max_accepted LONG DAYS 106751991 VALUE 294247-01-10T00:00:00.000Z +days_positive_overflow LONG DAYS 106751992 INVALID +days_min_accepted LONG DAYS -106751991 VALUE -290308-01-01T00:00:00.000Z +days_negative_overflow LONG DAYS -106751992 INVALID +instant_negative_fraction INSTANT -1 999999999 VALUE 1969-12-31T23:59:59.999Z +instant_epoch INSTANT 0 0 VALUE 1970-01-01T00:00:00.000Z +instant_micro_max INSTANT 9223372036854 775807000 VALUE 294247-01-10T04:00:54.775Z +instant_micro_min INSTANT -9223372036855 224192000 VALUE +instant_above_micro_max INSTANT 9223372036854 775808000 INVALID +instant_below_micro_min INSTANT -9223372036855 224191000 INVALID +instant_micro_min_plus_one INSTANT -9223372036855 224193000 VALUE -290308-01-01T19:59:05.224Z +instant_micro_max_fraction INSTANT 9223372036854 775807999 VALUE 294247-01-10T04:00:54.775Z diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv new file mode 100644 index 000000000..f55c4c584 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv @@ -0,0 +1,49 @@ +# case_id input source_unit target_type expected_wire_value expected_sql_value +micro_micro_zero 0 MICROS TIMESTAMP 0 0 +micro_micro_neg_1001 -1001 MICROS TIMESTAMP -1001 -1001 +micro_micro_neg_1000 -1000 MICROS TIMESTAMP -1000 -1000 +micro_micro_neg_999 -999 MICROS TIMESTAMP -999 -999 +micro_micro_neg_one -1 MICROS TIMESTAMP -1 -1 +micro_micro_pos_one 1 MICROS TIMESTAMP 1 1 +micro_micro_pos_999 999 MICROS TIMESTAMP 999 999 +micro_micro_pos_1000 1000 MICROS TIMESTAMP 1000 1000 +micro_micro_pos_1001 1001 MICROS TIMESTAMP 1001 1001 +micro_micro_min -9223372036854775808 MICROS TIMESTAMP -9223372036854775808 NULL +micro_micro_max 9223372036854775807 MICROS TIMESTAMP 9223372036854775807 9223372036854775807 +nano_nano_zero 0 NANOS TIMESTAMP_NS 0 0 +nano_nano_neg_1001 -1001 NANOS TIMESTAMP_NS -1001 -1001 +nano_nano_neg_1000 -1000 NANOS TIMESTAMP_NS -1000 -1000 +nano_nano_neg_999 -999 NANOS TIMESTAMP_NS -999 -999 +nano_nano_neg_one -1 NANOS TIMESTAMP_NS -1 -1 +nano_nano_pos_one 1 NANOS TIMESTAMP_NS 1 1 +nano_nano_pos_999 999 NANOS TIMESTAMP_NS 999 999 +nano_nano_pos_1000 1000 NANOS TIMESTAMP_NS 1000 1000 +nano_nano_pos_1001 1001 NANOS TIMESTAMP_NS 1001 1001 +nano_nano_min -9223372036854775808 NANOS TIMESTAMP_NS -9223372036854775808 NULL +nano_nano_max 9223372036854775807 NANOS TIMESTAMP_NS 9223372036854775807 9223372036854775807 +nano_micro_pos_fraction 999 NANOS TIMESTAMP 0 0 +nano_micro_zero 0 NANOS TIMESTAMP 0 0 +nano_micro_pos_one 1 NANOS TIMESTAMP 0 0 +nano_micro_pos_exact 1000 NANOS TIMESTAMP 1 1 +nano_micro_pos_over 1001 NANOS TIMESTAMP 1 1 +nano_micro_neg_fraction -999 NANOS TIMESTAMP 0 0 +nano_micro_neg_one -1 NANOS TIMESTAMP 0 0 +nano_micro_neg_exact -1000 NANOS TIMESTAMP -1 -1 +nano_micro_neg_over -1001 NANOS TIMESTAMP -1 -1 +nano_micro_min -9223372036854775808 NANOS TIMESTAMP -9223372036854775 -9223372036854775 +nano_micro_max 9223372036854775807 NANOS TIMESTAMP 9223372036854775 9223372036854775 +micro_nano_min_valid -9223372036854775 MICROS TIMESTAMP_NS -9223372036854775000 -9223372036854775000 +micro_nano_zero 0 MICROS TIMESTAMP_NS 0 0 +micro_nano_neg_one -1 MICROS TIMESTAMP_NS -1000 -1000 +micro_nano_neg_999 -999 MICROS TIMESTAMP_NS -999000 -999000 +micro_nano_neg_1000 -1000 MICROS TIMESTAMP_NS -1000000 -1000000 +micro_nano_neg_1001 -1001 MICROS TIMESTAMP_NS -1001000 -1001000 +micro_nano_below_min -9223372036854776 MICROS TIMESTAMP_NS +micro_nano_max_valid 9223372036854775 MICROS TIMESTAMP_NS 9223372036854775000 9223372036854775000 +micro_nano_pos_one 1 MICROS TIMESTAMP_NS 1000 1000 +micro_nano_pos_999 999 MICROS TIMESTAMP_NS 999000 999000 +micro_nano_pos_1000 1000 MICROS TIMESTAMP_NS 1000000 1000000 +micro_nano_pos_1001 1001 MICROS TIMESTAMP_NS 1001000 1001000 +micro_nano_above_max 9223372036854776 MICROS TIMESTAMP_NS +micro_nano_min -9223372036854775808 MICROS TIMESTAMP_NS +micro_nano_max 9223372036854775807 MICROS TIMESTAMP_NS diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv new file mode 100644 index 000000000..789bd6277 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv @@ -0,0 +1,21 @@ +# case_id input expected +null +empty +nil 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +all_fs ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff +upper_mixed 01234567-89AB-cDeF-0123-456789aBCDef 01234567-89ab-cdef-0123-456789abcdef +distinct_limbs 01234567-89ab-cdef-fedc-ba9876543210 01234567-89ab-cdef-fedc-ba9876543210 +both_min_null 80000000-0000-0000-8000-000000000000 +high_min_only 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +low_min_only 00000000-0000-0000-8000-000000000000 00000000-0000-0000-8000-000000000000 +bad_hex_high_first g1234567-89ab-cdef-0123-456789abcdef +bad_hex_high_second 01234567-x9ab-cdef-0123-456789abcdef +bad_hex_high_third 01234567-89ab-xdef-0123-456789abcdef +bad_hex_low_first 01234567-89ab-cdef-x123-456789abcdef +bad_hex_low_second 01234567-89ab-cdef-0123-x56789abcdef +short_java_uuid 1-1-1-1-1 +leading_space 01234567-89ab-cdef-0123-456789abcdef +trailing_space 01234567-89ab-cdef-0123-456789abcdef +unicode_hex ０1234567-89ab-cdef-0123-456789abcdef +wrong_dash 01234567_89ab-cdef-0123-456789abcdef +missing_dash 0123456789ab-cdef-0123-456789abcdef diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv new file mode 100644 index 000000000..4ad4ad323 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv @@ -0,0 +1,17 @@ +# case_id input_lo input_hi target_type expected_wire_type expected_utf8_hex expected_sql_value +uuid_string_asym -4941174770667877871 -6850330615249940744 STRING VARCHAR 61306565626339392d396330622d346566382d626236642d366262396264333830613131 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +uuid_string_zero 0 0 STRING VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303030 00000000-0000-0000-0000-000000000000 +uuid_string_one 1 0 STRING VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303031 00000000-0000-0000-0000-000000000001 +uuid_string_ones -1 -1 STRING VARCHAR 66666666666666662d666666662d666666662d666666662d666666666666666666666666 ffffffff-ffff-ffff-ffff-ffffffffffff +uuid_string_lo_min -9223372036854775808 0 STRING VARCHAR 30303030303030302d303030302d303030302d383030302d303030303030303030303030 00000000-0000-0000-8000-000000000000 +uuid_string_hi_min 0 -9223372036854775808 STRING VARCHAR 38303030303030302d303030302d303030302d303030302d303030303030303030303030 80000000-0000-0000-0000-000000000000 +uuid_string_both_min -9223372036854775808 -9223372036854775808 STRING VARCHAR +uuid_string_mixed 81985529216486895 -81985529216486896 STRING VARCHAR 66656463626139382d373635342d333231302d303132332d343536373839616263646566 fedcba98-7654-3210-0123-456789abcdef +uuid_varchar_asym -4941174770667877871 -6850330615249940744 VARCHAR VARCHAR 61306565626339392d396330622d346566382d626236642d366262396264333830613131 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +uuid_varchar_zero 0 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303030 00000000-0000-0000-0000-000000000000 +uuid_varchar_one 1 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303031 00000000-0000-0000-0000-000000000001 +uuid_varchar_ones -1 -1 VARCHAR VARCHAR 66666666666666662d666666662d666666662d666666662d666666666666666666666666 ffffffff-ffff-ffff-ffff-ffffffffffff +uuid_varchar_lo_min -9223372036854775808 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d383030302d303030303030303030303030 00000000-0000-0000-8000-000000000000 +uuid_varchar_hi_min 0 -9223372036854775808 VARCHAR VARCHAR 38303030303030302d303030302d303030302d303030302d303030303030303030303030 80000000-0000-0000-0000-000000000000 +uuid_varchar_both_min -9223372036854775808 -9223372036854775808 VARCHAR VARCHAR +uuid_varchar_mixed 81985529216486895 -81985529216486896 VARCHAR VARCHAR 66656463626139382d373635342d333231302d303132332d343536373839616263646566 fedcba98-7654-3210-0123-456789abcdef diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md new file mode 100644 index 000000000..187b7514d --- /dev/null +++ b/design/schema-aware-sender-journal.md @@ -0,0 +1,6928 @@ +# Schema-aware sender implementation journal + +North star: [schema-aware sender design](schema-aware-sender.md), revision 55 +(revision 9 at kickoff; source-backed clarifications recorded below). + +This journal records decisions, evidence, review outcomes and remaining work. +The design defines the intended product contract; this journal must not imply +that an unimplemented part already works. Corrections are recorded explicitly. + +Current status: iterations 1 through 2.36 are accepted for their recorded scopes, including typed +microsecond/nanosecond conversion (D020), broader timestamp units and `Instant` +with the approved range/precision rules (D021), and text/symbol conversions +(D023). Boolean conversion and strict ASCII text-to-boolean parsing are accepted +in D024. D030 revises D022's sequencing: public Sender integration is accepted +on the unreleased branch; full conversion coverage still gates release. +FLOAT/DOUBLE input conversion is accepted under the approved D026 rules, +including consistent NaN missing values and mathematical integer range checks. +STRING-to-numeric conversion is accepted in D027, preserving target-specific +syntax and rejecting LONG text overflow under the approved exception. +BINARY targets and exception-safe string/binary append rollback are accepted +in D029. LONG/UUID/floating text output is accepted in D039–D041; LONG and +STRING input to ordinary timestamps is accepted in D042–D043. Timestamp-to-text +with checked normalization is accepted in D044. Native CHAR and STRING-to-CHAR +are accepted in D045; STRING-to-LONG256 is accepted in D046, STRING-to-GEOHASH +in D047, and all 18 logical native decimal-to-decimal pairs in D048. +LONG-to-decimal is accepted in D049, STRING-to-decimal in D050, +FLOAT/DOUBLE-to-decimal in D051 and native-decimal-to-text in D053. +Negotiated discovery, lookup/cache coordination, ACK/NACK feedback, pinned +schema framing and safe SF recovery/replay are implemented and tested. +Non-owning bindings provide local UUID, LONG-to-numeric, primitive/`Instant` +timestamp, text/symbol, boolean, FLOAT/DOUBLE, STRING numeric, BINARY-target and +native decimal-to-decimal, LONG/STRING/FLOAT/DOUBLE-to-decimal and native- +decimal-to-text conversion over caller-owned table buffers. +Review corrections are recorded in D016 and D019; the approved LONG null rule +is recorded in D013. + +Automatic row-boundary schema adoption and ordinary Sender conversion are +implemented and tested in iteration 2.15. The original 47-method failure ledger +is fully resolved after iteration 2.36, but the complete feature is not yet +releasable because the wider conversion inventory and release-level +compatibility validation remain open. +D030 records the original 58, D032 the 56-test baseline, D033 the 54-test +inference result and D034 the current diagnostic repair. +The 2.15 scoped gates passed 410 client tests plus two packaging +checks, 222 server tests and three seeded repeats of the eight new E2Es. +Ordinary Sender can recover extended backlog and enforce its required +capability. Remaining conversion and release-validation work stays explicit +in the design's implementation-status section. + +Iteration 2.16 validates file-backed process-restart recovery through ordinary +Sender without manually seeded frames. The final server gate passed 230 tests; +the two new recovery cases also passed three seeded repeats. Client production +code and the accepted 2.15 artifact are unchanged. D031 records scope and evidence. + +Iteration 2.17's first repair group is accepted: independent legacy conversion +comparisons are restored, with 251 scoped tests and three repeats of the 25 +affected tests passing. The broad 333-test selection still has 56 failures/errors; +functional fixes remain ahead. D032 separates fixture, product and environment +evidence and records the reviewed inference plan. + +Iteration 2.18 implements confirmed-missing inference for supported setters using +the existing binding and buffer. The final gates passed 414 client tests plus +two packaging checks, 261 server tests and three seeded repeats of 16 integration +and recovery tests. The original 333-test selection has 54 failures/errors, with +two removals and no new failing methods. D033 records scope and evidence. + +Iteration 2.19 repairs seven stale diagnostic expectations without production +changes. The 268-test server gate and three repeats of 23 integration/recovery +cases passed. The original 333-test selection has 47 failures/errors, with +exactly seven removals and no additions. D034 records independent legacy +rejection coverage, typed local recovery, evidence and remaining decisions. + +Iteration 2.20 characterizes LONG/UUID null-sentinel text behavior without +production changes. Three new tests and the 271-test server gate pass, as do +three repeats of 26 integration/recovery cases. The broad 47-method failure set +is unchanged. D035 corrects the predicted LONG text result and records a +proposed, not yet approved, source-null policy for these five conversions. + +D036 records the [remaining-failures plan](schema-aware-sender-remaining-failures-plan.md), +including all 47 baseline methods and the source-backed distinction between +nine test repairs and 38 missing-conversion cases. No further implementation +iteration or new test run is accepted by this planning entry. + +Iteration 2.21 accepts phase 1a: five expectation/fixture repairs and one new +public Sender floating-boundary test. The 277-test server gate and three +32-test repeats pass. The original 333-test diagnostic now has 42 failures, +with exactly the five planned removals and no additions. Production code and +the client artifact are unchanged. D037 records acceptance and remaining work. + +Iteration 2.22 accepts phase 1b: four direct-buffer fixtures now send explicit +legacy frames with exact wire and original stored-value assertions. The 281-test +gate and three 36-test repeats pass. The original 333-test diagnostic has 38 +failing methods, with exactly the four planned removals and no additions. +All nine phase-1 test repairs are complete; production code and the client +artifact remain unchanged. D038 records acceptance and the conversion backlog. + +## Working agreement — 2026-09-09 + +The primary agent owns architecture, scope, integration and acceptance. SOL +subagents receive bounded implementation, source-validation and independent +review assignments. Authors self-review; a separate reviewer and adversarial +reviewer challenge important changes before an iteration is accepted. Reviews +are evidence, not votes: findings must be resolved or explicitly justified. + +Each iteration follows this loop: + +1. State the smallest observable behavior being delivered and its non-goals. +2. Verify load-bearing assumptions in current source and existing tests. +3. Delegate implementation with explicit file ownership and acceptance tests. +4. Run the real client/server E2E path and relevant regression tests. +5. Review correctness, failure sequences, simplicity and developer experience. +6. Record findings, fixes, exact validation and what the iteration did not prove. + +Prefer existing components and direct control flow. Do not introduce a framework +for a single use case. Keep producer state producer-owned and replay immutable. +Review allocations, synchronization and repeated work on the steady-state write +path, but do not claim performance from inspection alone. Changes to a hot path +need a measured, reproducible comparison before performance claims are accepted. + +Tests use public API outcomes, protocol contracts, deterministic SQL results and +delivery behavior. No reflection, private-field inspection, test-only production +hooks, visibility changes solely for tests or implementation call-count checks. +Use bounded event-based coordination. A mocked peer is useful for wire faults, +not a replacement for real-server ingestion tests. + +No commits, pushes or external coordination are part of the current instruction. +Preserve pre-existing work; serialize builds sharing generated outputs. + +## Carried-forward product decisions + +- Early detection means converting supplied values against the server schema + before buffering, not merely comparing setter types or an acceptance mask. +- Keep QWP v1. New clients work with old servers in legacy mode and always + request schema support. Confirmation enables schema mode without an opt-out. +- Upgrade is one-way for a sender. Subsequent connections must support the + extension. No send-time transformation, downgrade encoder or SF rewriting. +- An in-progress row and completed blocks retain their encoding snapshot. + Adopt new schema or legacy-to-schema mode only at a real row boundary. +- A local conversion/lookup failure cancels the partial row, not completed + rows. A refreshed schema can require a whole-row retry; never splice a new + converter into an already partially converted row. +- Server DDL and auto-creation races remain server-authoritative. Metadata + feedback is not acceptance, permission to retry a terminal NACK or ACK progress. +- Schema mode uses target missing-value semantics; legacy mode preserves the + existing behavior. Conversion parity for supplied values needs shared vectors. +- Ingestion schema discovery follows write authorization, not SELECT permission. +- Schema caches grow on demand, with the agreed million-entry ceiling. No + subscriptions, polling or eager million-entry allocation. +- A real E2E test works from the first iteration. Negotiation, conversion + families, schema evolution and recovery are added and tested incrementally. + +## D001 — Start with the existing real-server test infrastructure + +Date: 2026-09-09. Status: accepted; iteration 1 completed and verified. + +Iteration 1 establishes a small public `Sender` → real QuestDB → SQL baseline +that subsequent schema work can extend. Use the sibling server's existing QWP +E2E harness, not a new embedded server framework in the standalone client. +Production wire and conversion behavior remain unchanged in this iteration. + +The baseline must exercise a precreated target schema and matching typed values. +Additional cases should establish meaningful row/flush and local-error recovery +contracts, without pretending that local STRING-to-UUID conversion exists yet. + +Reason: a green path through the actual client artifact, transport, WAL ingestion +and SQL is the earliest useful integration gate. It is not proof of the proposed +schema extension. Existing adjacent tests remain part of validation. + +Starting workspace: + +- Client: `/home/jara/devel/oss/java-questdb-client`, HEAD `981bdb02`. + Pre-existing untracked file: `design/schema-aware-sender.md`. +- Server: `/home/jara/devel/oss/questdb`, HEAD `12a33d651e`. + Pre-existing untracked directory: `core/rust/qdbr/parquet2/`; out of scope. +- Toolchain: Corretto JDK 25.0.4, Maven 3.9.11, Linux amd64. +- `/tmp` has limited free space. Use task-owned temporary test storage on the + workspace filesystem if needed; do not delete unrelated temporary files. + +Assignments: + +- `baseline_implementation` (SOL): implement the narrow real-server E2E baseline; + no production or build-configuration edits. +- `architecture_validation` (SOL): verify source assumptions and recommend the + smallest subsequent lookup/binding slice; read-only initially. +- `adversarial_review` (SOL): challenge failure sequences and compatibility + assumptions independently; read-only. +- Primary architect: journal, build/artifact provenance, scope and acceptance. + +## D002 — Test the actual standalone client artifact + +Date: 2026-09-09. Status: accepted and verified. + +The server also contains a separate `java-questdb-client` checkout. Do not assume +that source is the client under test. Build/install the standalone workspace's +`questdb-client:1.3.10-SNAPSHOT`, then run the server tests without activating its +`local-client` reactor profile. Check the packaged manifest, artifact hash and +Surefire classpath. No source copies or build-configuration edits are needed. + +Verified artifact SHA-256 (workspace JAR and installed JAR match): +`b9f547437a66825e11da10d31bd89b8e4092c49c30b4f2ed4b09bc9af58df969`. +Manifest commit: `981bdb02a471f3b290c89b8e78cbc422610e329e`. + +## D003 — Interpret review findings against iteration scope + +Date: 2026-09-09. Status: accepted. + +The adversarial review confirms that current code does not yet have first-write +mode selection, immutable schema generations or lookup gating. These are planned +production changes, not defects introduced by the baseline test. Do not inflate +the first iteration to implement all of them. + +Two findings do refine the next brief: + +- Minimal request/confirmation classification must accompany the first real + schema lookup. A lookup cannot be sent to an unconfirmed peer. Milestone 3 + adds the broader compatibility/transition matrix; it does not authorize an + unsafe handshake shortcut in milestone 2. +- A matching-type SQL success cannot demonstrate that schema lookup/binding + occurred. The first schema slice must also observe the wire contract and a + controlled lookup failure at the public setter, before any row is accepted. + +Do not add a generic rollback framework merely because a reviewer suggested +one. Establish the required A/invalid-B/C behavior across generations first; +use the smallest producer-owned state transition that satisfies it. + +## D004 — Make duplicate semantics explicit per mode + +Date: 2026-09-09. Status: source-verified; implementation remains future work. + +`QwpTableBuffer.getOrCreateColumn()` calls `lookupColumn(name, type)` before the +already-written check. `lookupColumn()` checks the bound type first. Thus current +legacy behavior ignores same-type duplicates but rejects a different setter +type, even when the column already has a value in that row. + +Schema mode's first-value-wins rule must check whether the target column was +already supplied before converting a second value, including another setter +type. Legacy mode must preserve its existing type-sensitive behavior. Add +black-box tests for both, including a valid UUID followed by an ignored malformed +UUID string in schema mode. Do not describe this as universally unchanged +duplicate behavior. + +SOL updated the design to revision 10 with this distinction and D003's minimal +confirmation prerequisite. These are source-backed clarifications of the agreed +modes, not a new opt-out, downgrade path or expansion of the first iteration. + +## Iteration 1 evidence and review + +Implemented by SOL: one method in the existing server `QwpSenderE2ETest`, +`testTypedUuidPreservesCompletedRowAfterLocalError`. The scenario is precreated +UUID target, completed A, partially written B with a local type error, explicit +flush, valid C omitting B's marker, flush/close, exact SQL containing only A/C. + +Executed from the standalone client workspace: + +```sh +mvn -B -pl core -Dtest=QwpTableBufferTest,QwpWebSocketEncoderTest,QwpWebSocketSenderTest -Dmaven.javadoc.skip=true install +``` + +Result: PASS, 151 focused unit tests and 2 `JarPackagingIT` tests, no failures, +errors or skips. The resulting snapshot is installed for the server E2E run. + +Executed from the server workspace: + +```sh +mvn -B -pl core '-Dtest=QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketTypeConversionE2ETest,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-tests.lOeKVW -Dmaven.compiler.showWarnings=false test +``` + +Initial result: FAIL, 59 tests, 1 setup error, no assertion failures or skips. +All 58 existing adjacent tests passed. The new test's +`connectWs(port, 0, 0, 0)` configuration was rejected with +`disabling auto-flush is not supported for WebSocket protocol`. + +Reflection: author and architect both assumed that the helper's all-off +translation was a supported configuration. The real artifact test disproved +that assumption before any schema code was added. Fix the test through supported +public configuration, keeping A demonstrably buffered at the local error. Do not +change production configuration rules or rely on timing to make the test pass. + +The corrected public configuration uses row threshold 2, disabled byte trigger +and the largest accepted interval (`Integer.MAX_VALUE - 1` milliseconds). Only A +completes before the first flush; B fails in its setter. Review the actual +interval rule, not the helper's apparent ability to disable every trigger. + +Second run of the same 59-test selection: 58 adjacent tests passed; the new +test reached SQL verification but failed `Expected no timestamp but found ts, +idx=2`. The stronger `.returns(...)` assertion also validates cursor metadata; +the query needs an explicit designated-timestamp expectation. Keep that stronger +assertion rather than reverting to `.returnsOnce(...)`. + +Independent and adversarial source reviews accept the A/B/C test shape and +confirmed meaningful leak detection and UUID limb-order coverage. They did not +catch the missing query metadata expectation. Source-review approval therefore +does not complete the acceptance gate; executed tests remain mandatory. + +Architect review adds one further discriminator: use public +`flushAndGetSequence()` results to prove that each explicit flush actually +publishes data. Final SQL after `close()` alone could conceal a no-op flush. +Assert a nonnegative first FSN and increasing second FSN, not an exact number +or internal call count. This also checks the claim that A was still buffered. + +Final focused command uses the same server build options above with +`-Dtest=QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError`. +Result: PASS, 1 test, no failures, errors or skips. The actual test was compiled +and executed against the freshly installed standalone client artifact. + +Both SOL reviewers accepted the final test independently after the FSN and +timestamp assertions were added. The final architecture check confirmed those +FSN comparisons are public publication semantics, not internal counters. The +adversarial check distinguishes publication (FSNs), delivery (close/ACK drain) +and stored state (SQL after WAL application). + +Final regression rerun used the same 59-test selection and temporary directory, +with `surefire:test` after the final focused `test` command had compiled the +current sources. Result: PASS, 59 tests, no failures, errors or skips: 2 sender +E2E tests, 1 receiver-side local-error test and all 56 existing type-conversion +E2E tests. Log: server `core/target/schema-aware-tests.lOeKVW/final-regression.log`. + +SOL then ran three serial repetitions of the final compiled baseline through +`surefire:test`, using explicit `(fuzz.s0, fuzz.s1)` seed pairs `(1, 2)`, +`(42, 43)` and `(1234567, 7654321)`. Each run exited 0 with 1 test and no failures, +errors or skips. The primary architect independently checked the logs, including +the recorded seeds and test summaries. These vary the existing harness's seeded +fragmentation; they are not a claim of exhaustive fuzz coverage. + +Repeat command, substituting each seed pair: + +```sh +mvn -B -pl core '-Dtest=QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError' -Dfuzz.s0=1 -Dfuzz.s1=2 -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-tests.lOeKVW surefire:test +``` + +Logs are retained in the server's task-owned +`core/target/schema-aware-tests.lOeKVW/`: `final-focused.log`, +`final-regression.log` and `repeat-1.log` through `repeat-3.log`. + +Acceptance: iteration 1 is complete. The final gates are 151 client unit tests, +2 packaged-JAR tests, 59 selected real-server tests and 3 extra seeded baseline +runs, plus independent and adversarial SOL review. Only the single server test +method and the client design/journal changed. No production or build +configuration changes, commits, performance claims or claims of completed schema +behavior. The unrelated Rust directory remains untouched. + +## Next iteration candidate + +Schema lookup plus matching-type binding on the same real E2E path. Exact scope +and prerequisites will be selected from source validation and adversarial +findings. Do not implement the entire conversion matrix or rewrite replay as +prerequisites for that first schema-aware write. + +Source-backed constraints for the next implementation brief: + +- Start with one precreated LONG target plus its timestamp handling. Include + minimal handshake confirmation and a classified lookup failure at a setter; + successful matching-type SQL alone is insufficient evidence of schema use. +- Reuse the existing cursor I/O loop for a bounded control-request exchange. + No second query client/socket or shared-socket lock. Controls must not enter + SF or consume data FSNs; classify schema replies before ordinary ACK parsing. +- Obtain WAL metadata through the sequencer-aware engine metadata API, not a + reader snapshot that may lag WAL DDL. Authorize ingestion-schema disclosure + with table-write authorization before emitting it. +- Keep immutable lookup metadata separate from producer-owned target bindings. + Bind/cache at row/layout boundaries; avoid per-value wrapper allocations or + a generic converter hierarchy for the first matching-type case. +- Assign and review bounded control-frame encodings before implementation. + Full table identities, ACK feedback and conversion families remain later + slices; their absence must be explicit, with no premature release/advertisement + of the complete extension. + +## D005 — Subdivide discovery from high-level Sender activation + +Date: 2026-09-09. Status: accepted; supersedes the immediate LONG-binding +candidate above, not the final design contract. + +Source review exposed a staging problem with the proposed LONG-only activation: +once the high-level Sender enters schema mode, other existing conversion paths +must either gain converters, fail explicitly or silently pass through. The last +choice violates the design; implementing every converter would make this a large +iteration. Merely turning on negotiation without binding is not schema mode. + +Iteration 2.1 therefore implements the negotiated discovery control plane and +production client codecs, exercised over the real low-level WebSocket client +against the real server. The ordinary Sender does not request this capability +yet and retains its existing behavior. There is no user disable switch, fake +server, build-time test switch or test-only production hook. The full extension +is still unreleased and must not be advertised as complete. Activation remains +a subsequent milestone-2 increment with explicit supported-conversion behavior. + +This is a smaller increment than the initial commentary proposed. It delivers +real discovery, authorization and wire/lifecycle tests while keeping the existing +public Sender E2E path green; it does not claim setter-time detection yet. + +Assignments: + +- `schema_server_slice` (SOL): server handshake, discovery, bounded response + lifecycle and dedicated real-server E2E tests. +- `schema_client_slice` (SOL): low-level handshake opt-in, production request/ + response codec and immutable metadata, with contract tests. No Sender or SF + changes and no unused producer/I/O mailbox ahead of its first caller. +- `architecture_validation` (SOL): independent adversarial review of protocol, + authorization, sequence/ACK isolation, bounds and response backpressure. +- Primary architect: wire decisions, scope, integration, source verification, + journal and final acceptance; second review of all final diffs. + +## D006 — Keep discovery separate from data sequencing + +Date: 2026-09-09. Status: frozen for iteration 2.1 implementation. + +Use the ordinary QWP1 twelve-byte header and version 1, with a reserved +`FLAG_CONTROL = 0x20`, zero tables and exact payload length. No other flag may +be combined with CONTROL. This flag is not the future persisted `FLAG_SCHEMA`. + +All multibyte numbers are little-endian. Request payload: + +```text +kind:u8 = 1 (DESCRIBE), requestId:i64 > 0, nameByteLen:u16, tableName:utf8 +``` + +Response payload: + +```text +kind:u8 = 2 (SCHEMA), requestId:i64 > 0, result:u8 +result: 0 KNOWN, 1 MISSING, 2 DENIED, 3 UNAVAILABLE, 4 TOO_LARGE +KNOWN only: + tableId:i32, metadataVersion:i64, designatedIndex:i16, columnCount:u16 + repeated columnCount times: + nameByteLen:u16, name:utf8, serverEncodedType:i32, + paramsLen:u16, params:bytes +``` + +The server type is the full encoded server ColumnType integer, not the QWP wire +type or a masked base tag. It preserves existing timestamp, decimal, geohash and +array parameters without a second lossy mapping. Freeze this representation as +part of the schema-control contract. The separate length-delimited extension +parameter area is empty in this increment; unknown extension bytes can be +retained/skipped by the client. Future converters interpret known target types; +the transport codec does not reject a schema merely for an unfamiliar type. + +Bounds: frames at most 1 MiB, at most 2,048 active columns, at most 1,024 extension +parameter bytes per column, names at most 381 UTF-8 bytes and 127 UTF-16 units. +Validate UTF-8 strictly and follow existing identifier rules. The initial agent +suggestion of a 127-byte limit was rejected: it would exclude valid multibyte +names. Reject truncation, trailing bytes, conflicting flags and overflow before +allocation. Oversized schemas return TOO_LARGE, not MISSING. + +Skip dropped columns and remap the designated timestamp index into the compact +active-column list; -1 means no designated timestamp. Return schema identity and +columns from one sequencer-aware metadata snapshot. Authorize insertion before +opening/serializing known-table metadata; write-only users need no SELECT grant. +Only an absent table token yields MISSING; metadata-open/drop races yield +UNAVAILABLE. Denials carry no schema. + +Recognize control traffic before allocating a data message sequence. It must +not pass through data processing, WAL commit, data ACK bookkeeping or SF. A +dedicated parked schema-response state owns bounded bytes until fully sent; +completion resumes ordinary pending ACK/durable-ACK processing. Tests must +interleave controls and data/deferred writes and exercise fragmented responses. + +Low-level client API uses one-way `requestQwpSchema()` and +`isQwpSchemaEnabled()`; there is no high-level Sender option. Request and confirm +headers follow the design. Missing confirmation is unsupported; malformed +confirmation is an error. Ordinary clients request nothing new in this increment. + +Refinements accepted during implementation review: + +- The 1 MiB ceiling includes the QWP header. Effective response size is also + bounded by the existing raw send buffer, including its WebSocket header. + Precompute using overflow-safe arithmetic; return TOO_LARGE before allocating + or serializing a schema that cannot fit. Reuse the current send machinery; + no additional growable reply queue or application-level chunking in 2.1. +- Unknown length-delimited extension bytes are skipped, not allocated/stored + per column. Full encoded ColumnType already preserves all current parameters. + Remove the speculative `getColumnParams()` API until a real consumer needs it. +- Validate request table-name syntax, but do not apply setter syntax bans to + returned column names: authorized existing SQL metadata must round-trip. +- Recognized malformed controls close with WebSocket protocol error 1002, not + a data NACK or UNAVAILABLE. Detect the control flag as soon as its byte is + available, including a truncated header. Live read-only metadata lookup yields + UNAVAILABLE without mutating role-change/data-delivery state. + +## Iteration 2.1 evidence and review + +The following records implementation and review chronologically. Intermediate +green runs are not acceptance; final acceptance and executed gates appear below. + +Pre-build review findings being resolved: + +- Main review: the schema-send catch also enclosed subsequent ACK sends. An ACK + blocked by backpressure could therefore have its saved ACK state overwritten + with RESUME_SCHEMA. Restrict the catch to the schema send and test interleaving. +- Main and independent review: substring-based HTTP header scanning could treat + a prefixed header name or another header's value as schema confirmation. Match + whole header names at line boundaries and reject duplicate confirmations. +- Independent review: malformed-field tests wrote at incorrect byte offsets, + sometimes rejecting for an unrelated reason. Correct offsets and retain + targeted tests that fail if the particular bound is removed. +- Main review: a blanket catch(Throwable) hid VM/programming failures as schema + unavailability. Map expected lookup failures explicitly instead. +- Independent review: include the new schema-send state in close-time draining, + and apply the configured output-buffer cap before full schema serialization. + +These are findings in unfinished drafts, not accepted behavior. They must be +resolved and validated before the increment is marked complete. + +Interim evidence (not acceptance): the focused standalone client install passed +189 unit tests and 2 packaged-JAR tests, with no failures, errors or skips. The +primary architect verified the Maven summaries and that workspace/installed JAR +SHA-256 values match: +`546873e4d03b9e98ba24aa3b9d3a25ed793369a8ab30d4aedb9861b1752feb61`. +Log: client `core/target/schema-client-2.1-rerun.RrLlpw/focused-install.log`. +The installed artifact contains the final `RESULT_DENIED` API and skips unknown +extension parameters. Manifest commit remains the working-tree base `981bdb02`; +the hash, not that base commit alone, identifies the changed artifact. + +Independent SOL source review accepts both production diffs after the response +state, close-draining, header parsing and exception-mapping corrections. The +architect's second review requested two additional codec tests: every truncated +prefix with a repaired envelope length, and the maximum multibyte response name. +Dropping a byte while retaining the old payload length only tests envelope +rejection; it does not demonstrate safety at each deeper field boundary. + +The first real discovery E2E run exposed invalid CREATE TABLE syntax in the +new test, not a production failure. After adding `PARTITION BY DAY`, the live +negotiated request/response passed. An extended rerun also passed immediate WAL +ALTER metadata freshness and full encoded DECIMAL type. Authorization, +malformed/unnegotiated traffic, deferred-commit isolation and forced response +backpressure remain mandatory before acceptance. + +Additional bounded-allocation review: do not size the temporary encoded-name +array from historical metadata column slots, which include tombstones. Count +active columns first, reject above 2,048, and allocate only for that bounded +active set. Compute the compact timestamp index during that count. This is a +resource bound, not a measured performance claim. + +The extra client codec tests passed: 191 focused unit tests, no failures, errors +or skips. Log verified by the architect: +`core/target/schema-client-review.oAb6mA/focused-test.log`. This was a test-only +rerun (`test`, not `install`); the installed production artifact is unchanged. +An agent initially reported a nonexistent log directory; the architect checked +the filesystem and corrected the evidence before recording it here. + +Interim server regression passed 96 tests, no failures, errors or skips, against +the installed standalone artifact. The architect checked the Surefire classpath +and per-class summaries in server +`core/target/schema-aware-discovery.compile/final-regression.log`. This includes +three discovery tests and 93 adjacent tests, but not the still-unwritten +authorization and interleaving cases. A filename containing `final` does not +make this the final acceptance run. + +Test-plan reflection: the initial interleaving proposal reused a large network +relay. The production low-level encoder and WebSocket client can express +DATA → DESCRIBE → commit directly, so a relay is unnecessary for that case. +Also, the existing configured send-fragmentation path deterministically parks +the response when it exceeds the chunk size, even with a writable kernel socket. +Use those existing seams for schema-response resume. Testing an ACK blocked +immediately after a fully sent schema response additionally requires a small +schema response and deterministically coalesced input; merely sending two frames +quickly is not evidence that the server processed them in one read. + +Authorization E2E passed two tests over the real client/server connection: +write-only discovery succeeds despite SELECT denial, and INSERT denial returns +DENIED with no schema identity/columns. Log checked by the architect: server +`core/target/schema-acl.z7FCbt/acl-e2e.log`. The fixture uses the existing +SecurityContextFactory boundary; no production hook or visibility change. +Review tightened the SELECT-denial precondition to an authorization exception, +and ensured helper-created clients close if their handshake fails. + +Review of the first state-test draft found two false-positive risks despite its +green run. A schema-response barrier proves the server handled the preceding +frame, but does not prove a wrongly committed WAL transaction was applied; +drain WAL before asserting zero visible rows. Also, chunk-size 1 proves schema +send resumption but cannot exercise an ACK blocking inside the initial schema +handler: the schema itself always parks first. Keep separate tests for these +distinct response states and require the observed ordering appropriate to each. + +The complete new schema matrix subsequently passed 11 tests: six discovery/ +protocol/deferred/backpressure cases, two authorization cases and three bounds/ +read-only cases. Log verified by the architect: server +`core/target/schema-aware-discovery.compile/full-schema-matrix-final.log`. +Public SQL creation of 2,049 columns succeeds in this checkout, so the active +column-limit test reaches discovery's TOO_LARGE result without internal metadata +fabrication. A 512-byte send buffer rejects a wider response while allowing a +subsequent small lookup on the same connection. A controlled live read-only +engine boundary yields UNAVAILABLE, then KNOWN after re-promotion; this is not +an exhaustive endpoint-failover test. + +The initial coalescing socket fixture failed test compilation because its +constructor exposed SLF4J through a module the server tests do not read. The +final response-order fixture uses a small raw loopback WebSocket peer with +production QWP encoding/decoding. No build configuration was changed. An early +two-write fixture also allowed the ACK to arrive before the control; the final +fixture writes the combined pipeline and explicitly asserts SCHEMA → ACK → +next SCHEMA. One TCP write is not a universal one-read transport guarantee; +the observed response order is the test's discriminator. + +## Iteration 2.1 acceptance — 2026-09-09 + +Accepted after SOL implementation, independent/adversarial source review, +cross-review of the final test fixtures, and the architect's source/evidence +checks. No outstanding blocker remains within this increment's scope. + +Client commands (standalone client workspace): + +```sh +mvn -B -pl core -Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest -Djava.io.tmpdir=core/target/schema-client-2.1-rerun.RrLlpw -Dmaven.javadoc.skip=true install +mvn -B -pl core -Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest -Dmaven.javadoc.skip=true -Djava.io.tmpdir=core/target/schema-client-review.oAb6mA test +``` + +The install passed 189 unit tests plus 2 packaging tests; the final test-only +additions raised the focused unit selection to 191, all passing. Logs and the +unchanged installed artifact hash are recorded above. JDK Unsafe/removal and +native-access warnings were present; no build failure was hidden by them. + +Fresh final server regression, after restoring the correct production source: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-discovery.compile test +``` + +Result: PASS, 104 tests, no failures, errors or skips. This used the full `test` +lifecycle to compile current main/test sources, without the `local-client` +profile. The architect checked the standalone installed-JAR classpath and log +`core/target/schema-aware-discovery.compile/final-regression-104.log`. + +Three additional serial repetitions used the freshly compiled sources: + +```sh +mvn -B -pl core -Dtest=QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest -Dfuzz.s0=1 -Dfuzz.s1=2 -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-discovery.compile surefire:test +``` + +Seed pairs `(1,2)`, `(42,43)` and `(1234567,7654321)` each passed all 11 tests, +with no failures, errors or skips. Logs: `schema-repeat-1-2.log`, +`schema-repeat-42-43.log`, `schema-repeat-1234567-7654321.log` in the same server +directory. These are stability checks, not exhaustive fragmentation coverage. + +Adversarial mutation check: temporarily move the pending ACK pumps back into +the schema-send catch. The completed-schema/chunk-24 test then fails as intended: +the next DESCRIBE receives a duplicate 11-byte ACK rather than SCHEMA. The +observed exception is `Malformed QWP schema response: invalid frame length: 11`. +Log: `mutation-ack-catch.log` in the same directory. Restore with the exact inverse +patch before the final regression. The restored upgrade processor's SHA-256 +matches its pre-mutation value, independently checked by the architect: +`ec0e15bc6c8cc79be96f4a39152ad302596772f6a8f0996daf4ddd9147e5c9c0`. +This confirms the test detects the reviewed state-loss bug, not just happy-path +response delivery. No deliberate mutation remains in the workspace. + +Scope at handoff: discovery control frames and low-level client negotiation/ +codecs are complete. Normal Sender activation, local target conversion, schema +bindings, ACK/NACK schema feedback, persisted framing and replay changes remain +future work. No send-time transformation, opt-out, subscription/cache framework +or public feature-completion claim was introduced. No performance improvement +is claimed. No commits or pushes were made; unrelated Rust files and the accepted +iteration-1 baseline were preserved. Both repositories pass `git diff --check`. + +## Next increment + +Integrate lookup with producer-owned immutable schema bindings and a first real +setter-time failure E2E. Select the activation boundary and supported conversion +slice from source before changing the high-level Sender. Do not silently pass +through conversions merely to enable negotiation, and do not expand this into a +send-time transformation or replay rewrite. + +## D007 — Test the conversion component before Sender activation + +Date: 2026-09-09. Status: accepted and verified in iteration 2.2. + +Do not activate normal Sender merely to exercise a converter. Its flush path +encodes and durably appends bytes before I/O sends them. Activating schema mode +therefore also needs coherent first-use negotiation, row boundaries, frame +identities, reconnect and persisted-backlog compatibility. Adding FLAG_SCHEMA +but deferring its recovery checks would violate the design's first-framing-change +gate. Implementing all of that alongside the first converter is too large a +single increment. + +Instead add the genuine production `QwpSchemaRowBuffer` component, holding one +immutable server snapshot and a private sparse existing `QwpTableBuffer`. Its +first supported pairs are STRING-to-UUID, typed UUID-to-UUID and matching LONG. +Unknown targets, missing columns and unsupported non-null pairs fail explicitly, +not through raw-value fallback. Null string input is an effective target NULL +for the supported targets, matching current QWP setter semantics. `atNow()` +retains server-assigned time; explicit timestamp conversion is not in this slice. + +The E2E path is real DESCRIBE → immutable binding → locally converted A / invalid +B / valid C → existing typed UUID/LONG QWP encoding → real server ACK and SQL. +Observe typed UUID bytes, not merely successful server-side parsing. Failures +cancel B and its newly introduced buffer columns, not completed rows. Duplicate +suppression uses the target column before validating/converting a second input, +including cross-setter duplicates. Keep the mutable storage private and expose +only row operations and encoding; no per-value wrapper allocation or new queue. + +This is deliberately conversion-component E2E, not full schema mode. It does +not transmit schema identities, activate Sender, add a lookup mailbox/cache or +change SF. The existing type definitions faithfully describe the already +converted bytes; no STRING value is passed through as a substitute for UUID +conversion. Design revision 12 makes this staging distinction explicit. + +Reflection: initial agent briefs coupled any schema-bound component to the new +persisted flag. The architect challenged that coupling: wire identities enable +feedback and snapshot accounting, but are not required to test a standalone +local converter honestly. Independent review agreed after separating component +behavior from activation claims. Another brief proposed rejecting a stale +identity; this was rejected outright. The design says identity is descriptive, +not CAS, and a version mismatch alone must not reject otherwise valid data. + +Assignments: + +- `schema_client_slice` (SOL): row component, classified schema exceptions, + minimal UUID conversion helper if needed, codec refinement and component tests. +- `schema_server_slice` (SOL): real-server component/UUID conformance E2E tests; + no server production or framing changes. +- `architecture_validation` (SOL): independent/adversarial API, ownership, + rollback, unknown-parameter and test-discrimination review; read-only. +- Primary architect: scope, source verification, design/journal, integration, + artifact provenance and final acceptance. Build slots remain serialized. + +## D008 — Preserve unknown extension presence for the first real consumer + +Date: 2026-09-09. Status: accepted for iteration 2.2. + +Iteration 2.1 correctly skipped opaque per-column extension bytes without +allocating/storing them. A converter now needs one additional fact: whether +such bytes were present. Otherwise an unfamiliar extension on a familiar base +type could be silently ignored when writing that column. + +Retain a boolean per column and expose +`hasColumnExtensionParameters(index)`. Continue skipping the bytes and keep the +snapshot immutable. Reject use of a parameter-bearing column with +UNSUPPORTED_FEATURE; allow unrelated supported columns. Known parameters remain +packed in the full ColumnType integer. The first converter compares exact +supported types, not masked tags that could discard unfamiliar flags. This is +a consumer-driven refinement, not restoration of speculative byte arrays/API. + +## D009 — Compare UUID conversion with QWP ingestion, not SQL casts + +Date: 2026-09-09. Status: source-verified and runtime parity verified in iteration 2.2. + +Server `WalColumnarRowAppender.putStringToUuidColumn()` checks the null bitmap, +then calls strict UUID length/dash/hex parsing. An empty non-null string fails. +`SqlUtil.implicitCastStrAsUuid()` instead treats an empty string as null, so it +is not the correct conformance oracle. Java's UUID parser is not substituted. + +Reuse the narrow server parsing rules with the client's existing hexadecimal +parser. Parse both limbs before append. The public input order is `(lo, hi)`; +existing `ColumnBuffer.addUuid(hi, lo)` writes low then high. Preserve the native +sentinel limitation: both UUID limbs equal to Long.MIN_VALUE represent NULL; +a single such limb does not. Matching LONG likewise preserves its null sentinel. + +Use one versioned UUID vector corpus, vendored byte-identically into both test +resource trees, so each repository's tests are hermetic. Verify equal hashes. +Compare acceptance/rejection and stored values against actual legacy QWP string +ingestion as well as the new local component. Invalid input must never become +a successful null merely because a different SQL conversion would accept it. + +## Iteration 2.2 evidence and review + +Development evidence follows; final acceptance is recorded below. Do not infer +high-level Sender activation from component completion. + +Read-only baseline, against the previously installed 2.1 client and compiled +server sources: + +```sh +mvn -B -pl core '-Dtest=QwpWebSocketTypeConversionE2ETest#testStringToUuidColumn+testUuidToStringColumn+testUuidToVarcharColumn' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-binding-baseline.bAkuPn surefire:test +``` + +PASS: three tests, no failures, errors or skips. Log checked by the architect: +server `core/target/schema-binding-baseline.bAkuPn/uuid-baseline.log`. + +Draft review caught input-pair validation running before duplicate suppression. +Fix by resolving the actual target wire type, asking the existing sparse buffer +for that target column, and honoring its duplicate result before validating the +input pair. Do not introduce a parallel row-stamp array or another row tracker; +one existing bookkeeping source suffices. Other draft checks cover closed-state +reuse, table/column validation, ambiguous schema names and failure cleanup. + +Keep the design's five exception reasons. In this component, confirmed MISSING +means inference is not implemented, and TOO_LARGE means discovery cannot supply +the bounded schema; both are UNSUPPORTED_FEATURE with distinct diagnostics. +DENIED is ACCESS_DENIED and transient UNAVAILABLE is SCHEMA_UNAVAILABLE. +Do not invent staging-only public reason codes or call these INVALID_VALUE. +Reserved SCHEMA_CHANGED is not emitted until actual refresh/rebinding exists. + +Independent review also caught a test-client leak if the handshake failed before +the helper returned, and a conformance gap: only invalid strings initially went +through the legacy server converter. The helper now closes on construction +failure. The E2E test compares every accepted value, explicit null and native +sentinel through both local binary UUID encoding and actual server VARCHAR-to- +UUID ingestion; each invalid vector gets a fresh stream and must receive a NACK. + +First client build passed 200 unit tests and two packaging tests. The architect +checked the log and matching workspace/installed JAR hash. Before accepting the +slice, test review requested stronger wire assertions: an encoder returning a +positive length alone does not prove row retention, null encoding or rollback. +The server test must assert exactly the A/C rows and their effective columns, +so a failed-row-only column cannot remain unnoticed as an all-null definition. +Client tests must likewise inspect observable bytes after partial-row rejection, +cancel/reset and explicit nulls. Final evidence follows after these checks run. + +Server validation caught two test authoring mistakes before acceptance. The +first compile lacked required `WebSocketFrameHandler.onClose()` callbacks. The +first executed run then passed two of four tests; the other two expected `NaN` +for LONG nulls, while this server's SQL renderer returns `null`. The architect +checked both actual/expected outputs: the UUIDs, retained A/C rows and values +matched, and only the null text differed. Fix the expected text, not production +conversion semantics, and rerun the tests and regression selection. + +The client build also caught a server/client API difference (`NumericException` +uses `instance()` in this client, not `INSTANCE`) and a malformed test resource +with literal `\\t` text instead of actual tab separators. Both were corrected +before the successful install. Failure logs are retained as +`core/target/schema-row-buffer.2.2/focused-install.log` and +`focused-install-rerun.log`; the architect checked the actual failures. + +## Iteration 2.2 acceptance — 2026-09-09 + +Accepted after author self-review, independent SOL adversarial/source review, +architect source review and direct checks of build logs, artifact provenance +and corpus hashes. No concrete blocker remains within the component scope. + +Client install (200 unit tests plus two packaging tests, all passing): + +```sh +mvn -B -pl core -Dtest=QwpSchemaRowBufferTest,QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest -Djava.io.tmpdir=core/target/schema-row-buffer.2.2 -Dmaven.javadoc.skip=true install +``` + +Log: `core/target/schema-row-buffer.2.2/focused-install-final.log`. +After test-only strengthening, the same selection passed **203 tests** on final +sources with `test` instead of `install` and temporary directory +`core/target/schema-row-buffer-final.2.2`. Log: +`core/target/schema-row-buffer-final.2.2/final-focused-test.log`. +The intermediate component/codec-only run passed 21 tests; its log is +`core/target/schema-row-buffer-tests.2.2/strengthened-test.log`. +All green runs have zero failures, errors and skips. Production did not change +after installation; the final test-only changes did not require reinstallation. + +Workspace and installed `questdb-client-1.3.10-SNAPSHOT.jar` SHA-256: +`eeec24a05e186cb22c9b063008a5febf1bf5b57bd640d8fe9f24d82669d26251`. +The manifest records base commit `981bdb02a471f3b290c89b8e78cbc422610e329e`; +the hash, not that base commit alone, identifies this dirty-worktree artifact. +Server Surefire reports explicitly reference the installed standalone artifact +under `/home/jara/.m2/repository/org/questdb/questdb-client/1.3.10-SNAPSHOT/`. +No embedded-client profile was used. + +Fresh server regression, against server base +`12a33d651e51e2682e7a448c8db5168fc72dfad3` plus the task changes: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaRowBufferE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-row-buffer-server test +``` + +PASS: **108 tests**, zero failures, errors or skips. Log: +server `core/target/schema-row-buffer-server/final-regression.log`. +The four new E2E tests also passed separately in `new-e2e-final.log`. +Three additional serial runs used `-Dtest=QwpSchemaRowBufferE2ETest`, +`-Dfuzz.s0= -Dfuzz.s1=` and `surefire:test` against those compiled +sources. Seed pairs `(1,2)`, `(42,43)` and `(1234567,7654321)` each passed all +four tests. Logs: `repeat-1-2.log`, `repeat-42-43.log` and +`repeat-1234567-7654321.log` in the same server directory. These are stability +checks, not an exhaustive fuzzing or performance claim. + +The shared 20-case corpus is at test resource +`io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv` in each repository. +Both copies have SHA-256 +`c0ce3c5836cc58e1960dbcadbcfd9d53df0d842af796c76d25d942259cec90e4`. +Tests inspect public protocol bytes and deterministic SQL, without reflection, +private-state hooks or runtime cross-repository resource dependencies. + +Delivered scope: immutable schema-bound row construction, STRING-to-UUID, +typed UUID/LONG writes, explicit target nulls, duplicate-first semantics, +partial-row rollback and stable classified exceptions. Opaque extension presence +is retained without storing its bytes; unknown used types are rejected locally. +The mutable row storage remains private and reuses existing row bookkeeping. + +Remaining scope: ordinary Sender activation and lookup coordination, inferred +schemas, further conversion families, schema identities in persisted frames, +ACK/NACK schema feedback, snapshot transitions and one-way-upgrade/recovery +integration. Caller-owned response correlation and snapshot acquisition remain +explicit component preconditions. No send-time transformation, feature opt-out, +subscription machinery or parallel row-state tracker was introduced. +No existing Sender, I/O, encoder, SF or server production behavior changed in +iteration 2.2. Both worktrees pass `git diff --check`; unrelated work remains +preserved. No commits or pushes were made. + +Next architectural gate: define the smallest pinned-frame/recovery increment +before activating Sender. Introducing persisted schema framing must include +negotiation and replay/recovery checks in the same increment. Continue to keep +the real component and public-Sender baseline E2E paths green throughout. + +## D010 — Persisted schema framing and replay before Sender activation + +Date: 2026-09-09. Status: accepted in iteration 2.3; evidence below. + +After the user asked what unchanged ordinary Sender behavior meant, the +architect checked whether this increment could safely activate it. Source +review and independent SOL review found that UUID/LONG-only activation would +either reject the other existing conversions globally on supporting servers or +silently pass values through. The latter violates the design; the former is +not an acceptable ordinary-Sender regression. Do not use a per-table, SF or +configuration exception as an implicit feature opt-out. + +Complete the next prerequisite instead: extended table framing and safe SF +recovery. This is still not high-level schema activation, and is not described +as the final prerequisite: lookup coordination, conversion coverage, snapshot +transitions and feedback remain. The public Sender baseline stays green. + +Wire decision, captured in design revision 13: `FLAG_SCHEMA=0x02`; after each +table name, kind 0 means unknown, or kind 1 precedes a non-negative i32 table +ID and i64 metadata version, little-endian. Reject invalid kinds/fields, +SCHEMA+CONTROL and schema-flagged table-less frames. Reject unnegotiated schema +frames before data processing. Stale identities remain descriptive, not CAS. +ACK metadata feedback is not needed to parse/persist these identities and is +not implemented in this slice. + +The encoder gets explicit per-operation schema methods. Do not turn the +existing mutable flags setting into a sticky schema mode that can relabel +later legacy data. A message has one layout; reject mixing legacy/schema table +addition methods. Split messages retain the original flag and table bytes. +The schema row component passes its immutable snapshot identity to the encoder; +there is no new mutable identity field on QwpTableBuffer merely for serialization. + +Recovery derives its requirement in the existing ordered frame scan, before +any reconnect can send dictionary catch-up or a legacy prefix. Any unacknowledged +retained extended frame requires support, conservatively including a deferred +tail; acknowledged residue does not. Foreground and background/drainer paths +must enforce the requirement for their own slot on every connection. A mismatch +retains bytes and watermarks and must not become quarantine, stripping or +downgrade. Fresh ordinary Sender still does not request the extension in this +staging increment; schema-required replay does request it. No second SF scan or +new persistent side file is planned. + +Assignments: SOL client author owns explicit encoding and SF/reconnect safety; +SOL server author owns bounded parsing/ingress and real-server tests; independent +SOL reviewer challenges lifecycle, compatibility and test discrimination. +The architect owns the contract, journal, source checks and final acceptance. +Tests must cover actual disk recovery and byte-identical replay, not just flag +getters, and preserve the existing component and public-Sender E2E coverage. + +## Iteration 2.3 development evidence + +Pre-edit compiled baseline, run without recompiling while authors prepared the +new source changes: + +```sh +mvn -B -pl core '-Dtest=RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-framing-baseline.LGBRLN surefire:test +``` + +PASS: 27 tests, zero failures/errors/skips. Log: +`core/target/schema-framing-baseline.LGBRLN/baseline.log`. This establishes the +unchanged recovery baseline; it is not validation of the new source edits. + +Source review found local deferred-tail retirement in both BackgroundDrainer +and the cursor loop before any connection. The conservative schema-required +tail rule must gate that self-acknowledgment too, otherwise an incompatible +endpoint can coincide with watermark advancement and deletion without sending +any bytes. Keep legacy-tail retirement unchanged. A supporting connection may +still abort the orphaned transaction normally; tests must not expect deferred +rows without a commit to be ingested. + +Draft encoder review found transient mutation of configured flags around header +writing. A header allocation failure could leave FLAG_SCHEMA set for a later +legacy encode. Pass computed header flags explicitly instead, including the +existing delta-dictionary path. Reject schema/table-add layout mixing and +schema-flagged table-less construction; do not add another buffering framework. + +Independent and architect review caught schema-capability errors being routed +through the existing durable-ACK mismatch budget. That budget can quarantine an +orphan and report data loss, which is wrong for schema-required bytes. Schema +mismatch must retain/retry until stop or a compatible endpoint, and reset—not +consume—the unrelated durable-capability episode. Preserve typed errors through +endpoint exhaustion so the correct policy sees them. + +One-way support is owner-wide, not just a flag on an individual reconnect +factory. A new legacy-only orphan connection cannot go to an old server after +the owning sender required/confirmed schema support; the reverse ordering also +matters. Keep the recovered per-slot requirement and one shared monotonic sender +requirement. Existing legacy connections need not be forcibly closed solely for +another connection's upgrade, but every subsequent connection must respect the +shared requirement. Tests need both foreground/background orderings. + +The architect and independent reviewer also found a publication race: checking +the requirement only at the start of trySendOne allows the producer to publish +the first extended frame after that check. Recheck after observing the complete +published frame, before delta processing or send. The producer's requirement +write precedes ring publication, so this ordering makes the guard effective. +Keep the earlier guard for retirement/catch-up paths. Do not add private test +barriers to force this exact instruction interleaving; combine public live- +append coverage with the explicit memory-ordering argument. + +Structural cost budget: a known identity adds 13 bytes per table block, unknown +adds one byte, never per row. Recovery folds one boolean into its existing +ordered scan; it does not reread the SF log. Capability checks are per frame or +connection, not per value. These are source/protocol observations, not measured +throughput, latency or allocation-improvement claims. + +A custom reconnect factory may ignore the new default requireSchema callback. +The background drainer must check the returned connection inside its typed +retry loop, using the actual recovered engine requirement. Checking only at +send-loop startup would let a capability mismatch escape through generic setup +failure handling and mark the slot failed. Independent review accepted the +real-network regression: a factory returns old-server connections twice, stop +terminates retries, no frames or data-loss errors occur, and reopening the disk +slot proves both watermarks and schema-required backlog were retained. + +First integrated client install passed 294 unit tests and two packaging checks, +with zero failures, errors or skips. Log: +`core/target/schema-framing-final.2.3/focused-install.log`. The architect checked +the aggregate results and matching workspace/installed main-JAR SHA-256: +`6e1d9067594db8afd70c96d0cbd7b999eed56a18c1a16663c686d6bce1832340`. +This artifact unblocked fresh server compilation; it is not final acceptance. +Explicit owner-wide negotiation and split/deferred identity-preservation tests +are still required. The real-network tests already cover incompatible peers +that also lack durable ACK, exact mixed legacy/schema replay after recovery, +and deferred-tail retention followed by normal local abort on a supporting +connection. No new white-box production hooks were introduced. + +Fresh server validation against that standalone artifact passed 265 tests: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaIdentityParserTest,QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide,QwpUdpMalformedTest,QwpUdpInsertTest' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-identity-server test +``` + +Log: server `core/target/schema-identity-server/final-regression.log`. +The architect verified zero failures, errors and skips, and the standalone +`/home/jara/.m2/repository/org/questdb/questdb-client/1.3.10-SNAPSHOT/questdb-client-1.3.10-SNAPSHOT.jar` +in the identity E2E Surefire runtime classpath. The earlier fresh focused run +passed 61 tests; log `core/target/schema-identity-server/focused.log`. +Three further serial runs used `-Dtest=QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest`, +`-Dfuzz.s0= -Dfuzz.s1=` and `surefire:test` against those compiled +sources. Seeds `(1,2)`, `(42,43)` and `(1234567,7654321)` each passed nine tests. +Logs: `core/target/schema-identity-server/repeats/repeat--.log`. +These are stability checks, not exhaustive fuzzing. UDP regression is included +because the shared parser now understands schema framing; both ordinary and +Linux batched UDP receivers must reject it without WebSocket negotiation. + +Test review rejected version-zero/table-less pseudo-frames as positive schema +recovery fixtures and a fixed 100 ms checkpoint sleep. Use real encoder output +for both schema frames and the retained legacy suffix, with lengths matching +the allocations. Observe the public ACK watermark with a bounded wait before +reopening. Deliberately malformed delta-parser fixtures remain synthetic. +These corrections strengthen the evidence; a green run of the earlier fixtures +alone does not establish valid-frame recovery correctness. + +## Iteration 2.3 acceptance + +Date: 2026-09-09. Accepted by the architect after SOL author self-review, +independent source/adversarial review, and final-source runtime verification. +No remaining finding blocks the frozen scope. Both worktrees remain uncommitted; +unrelated server `core/rust/qdbr/parquet2/` was not touched. + +Final client command: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-framing-acceptance.2.3 -Dmaven.javadoc.skip=true test +``` + +PASS: **298 tests**, zero failures, errors or skips. Log: +`core/target/schema-framing-acceptance.2.3/focused-test.log`. +The preceding final-source run passed 297 before the live-append test was added; +log `core/target/schema-framing-final-source.2.3/focused-test.log`. +The two packaging checks passed during the earlier install recorded above. +Production did not change after that install; final edits were test-only. +The architect rechecked the matching workspace and installed main-JAR SHA-256: +`6e1d9067594db8afd70c96d0cbd7b999eed56a18c1a16663c686d6bce1832340`. +The manifest still records base commit +`981bdb02a471f3b290c89b8e78cbc422610e329e`; the hash identifies the dirty-worktree +artifact. Server validation remains the fresh **265-test** gate recorded above, +with three additional nine-test seeded E2E passes against this same artifact. + +The final five-test client network suite passed three more serial runs using +`-Dtest=QwpSchemaReplayNetworkTest`, `surefire:test` and +`-Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-framing-repeats.2.3/run-`. +Logs: `core/target/schema-framing-repeats.2.3/run-1.log`, `run-2.log`, `run-3.log`. +No seed or forced-interleaving claim applies to these repetitions. + +The last live-append test starts the actual cursor loop on an already-connected +legacy peer with an empty disk engine, then publishes the first valid extended +frame through appendBlocking. It observes repeated connection attempts, no +binary frames at the peer, no terminal/quarantine outcome, unchanged ACK and +publication watermarks, and retained schema-required backlog after reopening. +Together with the post-publication guard's memory-ordering argument, this covers +the important transition without a private barrier or new production test hook. +The acknowledged-residue fixture also explicitly proves FSN 0 remains represented +in the recovered segment before checking that it does not require schema mode. + +Delivered: immutable encoding identity survives table framing, splitting and +disk replay; negotiated server parsing accepts descriptive stale identities; +unnegotiated WS and all UDP ingress reject extended framing; foreground and +background reconnects share a one-way requirement; capability mismatch retains +the backlog rather than transforming, quarantining or acknowledging it away. +Existing legacy wire bytes and durable-ACK failure policy retain regression +coverage. No protocol-version bump, send-time transformation, feature switch, +second recovery scan or persistent capability side file was introduced. + +Reflection: persisted-format changes reach beyond parsing. Startup, orphan-tail +self-acknowledgment, mixed backlog prefixes, shared connection ownership and +producer/I/O publication all need the same safety contract. The small explicit +encoder API and existing recovery scan were sufficient; a parallel buffering or +recovery framework was unnecessary. Source review found the lifecycle mistakes +before acceptance; valid-frame and real-network tests now guard their outcomes. +No throughput or latency improvement was measured or claimed. + +Remaining product work is explicit: ordinary-setter activation, producer-owned +lookup coordination, inferred schemas, the remaining conversion families, +snapshot transitions, and ACK/NACK schema feedback. Next, inventory conversion +coverage against server source and choose a bounded family with real-server +conformance tests. Do not activate the UUID/LONG-only component globally or +introduce an implicit opt-out/raw fallback to make incomplete activation pass. + +## D011 — LONG-input numeric conversion family + +Date: 2026-09-09. Status: accepted in iteration 2.4 on 2026-09-10, with the +sentinel parity assumption corrected by D012 and the approved rule in D013. + +Three SOL discovery briefs proposed different slices: STRING/LONG to INT/LONG, +STRING/LONG to all integer widths, and LONG to all six scalar numeric targets. +The architect selected the last: extend the existing schema row component's +longColumn to BYTE, SHORT, INT, LONG, FLOAT and DOUBLE. This follows one coherent +server dispatch without introducing distinct string grammars or a private SHORT +parser in the same increment. The narrower text proposals remain useful future +work, not rejected product functionality. + +Source authority: server `cairo/wal/WalColumnarRowAppender.putIntegerToNumericColumn` +and `cutlass/qwp/protocol/QwpFixedWidthColumnCursor.isCurrentValueSentinelNull`. +The architect read both after the independent reviewer identified the shared +dispatch. Source LONG_NULL is classified before range checks or casts. Correct +the initial client brief's suggestion to treat every raw LONG as an ordinary +number: Long.MIN_VALUE must become target missing/null, not overflow or a large +negative FLOAT/DOUBLE. BYTE/SHORT store zero for null; INT/LONG use their null +sentinels and floating targets use NaN. A non-null LONG equal to Integer.MIN_VALUE +is within range and becomes the INT null sentinel after narrowing. Rejecting +that value would diverge from current QWP behavior. + +BYTE, SHORT and INT use inclusive Java signed ranges. LONG is identity. +FLOAT/DOUBLE use the same Java casts as ingestion; precision loss is accepted, +including 16,777,217 to float 16,777,216 and 9,007,199,254,740,993 to double +9,007,199,254,740,992. Encode the target wire type directly. No fallback that +leaves the server to perform the new conversion is permitted. + +Keep target-to-wire resolution small and shared so an already-written column +ignores duplicate setters before pair/range/value validation, including a +different existing setter on a newly supported target. The existing explicit +null behavior of stringColumn(null) may cover the new supported targets; +non-null numeric string parsing remains UNSUPPORTED_FEATURE unless the write +is an ignored duplicate. Used unknown/parameterized targets remain explicit +limitations, not INVALID_VALUE. Range failures cancel the entire partial row +and its new-only bindings, preserving completed rows through the existing +rollback machinery. No extra row-state tracker or converter framework. + +Assignments: SOL client author owns QwpSchemaRowBuffer, unit tests and canonical +shared vectors; SOL server author owns real-server tests and an identical +vendored resource, with no server production changes; independent SOL reviewer +challenges parity, sentinels, duplicate ordering, rollback and evidence quality. +Client and server Maven runs remain serialized within their repositories. Server +tests use the freshly installed standalone client, never the embedded-client +profile. The architect owns this journal, scope and final acceptance. + +Acceptance requires explicit expected wire values/types/null bits and SQL +results, not only equality between two paths. Exercise accepted and rejected +vectors through both local target-typed conversion and legacy raw LONG server +conversion; rejected server batches use an isolated connection. Include integer +boundaries and one-outside values, floating precision boundaries, source nulls, +target sentinel collisions, omissions, duplicate-before-overflow and A/invalid-B/C +rollback with B-only columns absent. Keep the accepted framing/recovery and +ordinary Sender E2E regressions green. Tests use public component/wire/SQL +contracts, no reflection, production test hooks or arbitrary sleeps. + +This is new functional coverage, not a performance optimization or bottleneck +claim. Check successful conversion paths for per-value object allocation and +unnecessary scans; do not claim measured throughput/latency improvements. +Deferred: non-null text parsing, floating-source conversions, timestamps, +decimals, arrays, all other families, lookup coordination and ordinary Sender +activation. Design revision 14 only clarifies the historical staging paragraph: +component E2E need not initially transmit identities, but accepted iteration 2.3 +now does. The component's stale framing Javadoc will be corrected with its edit. + +## Iteration 2.4 development evidence + +Pre-edit component baseline: + +```sh +mvn -B -pl core -Dtest=QwpSchemaRowBufferTest -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-baseline.2.4 -Dmaven.javadoc.skip=true test +``` + +PASS: 12 tests, zero failures/errors/skips. Log: +`core/target/schema-numeric-baseline.2.4/baseline.log`. Maven ran the normal +lifecycle but found compiled classes up to date; this checks the accepted +pre-edit component, not the new conversion implementation. + +The architect found a null-ordering bug in the first draft. Generalized wire-type +resolution also accepts UUID for the existing UUID setter; an unconditional +LONG_NULL shortcut therefore accidentally accepted LONG-to-UUID as UUID null. +Validate that the source/target pair is supported after duplicate suppression +but before interpreting the source null sentinel. Unsupported pairs remain +unsupported even for LONG_NULL. A duplicate LONG_NULL after a valid UUID value +must still be ignored. Dedicated regressions cover both outcomes and rollback. + +This differs from the intentional stringColumn(null) behavior frozen in D011: +that existing explicit-null operation is allowed on all supported numeric +targets. Non-null numeric strings are still unimplemented. Independent review +challenged the scope; the architect retained the documented target-owned null +rule and required null/omission coverage rather than introducing a source-type +restriction accidentally. + +The initial 29-vector corpus was expanded to 47 before freezing. It includes +ordinary values, signed boundaries, one-outside rejections, sentinel collisions, +positive and negative precision boundaries and LONG_MIN+1. FLOAT/DOUBLE expected +results are literal IEEE bit patterns, not production casts in the oracle. +The architect independently checked all 21 non-null floating expectations using +BigInt significands, discarded-bit comparison and ties-to-even rounding; all +matched, without using Java or JavaScript floating-point casts. The +token denotes the target missing representation, explicitly SQL zero for +BYTE/SHORT. Canonical client resource: `io/questdb/client/cutlass/qwp/long-to-numeric.tsv`. +Frozen SHA-256: `0b574bc3b53fb7cc027b0cd2a9fc8d63f75a99d35f6f7f206d7bd21a70334f64`. + +Review also requires tests that distinguish completed-row preservation from +new-only-column rollback: the invalid row must introduce a separate column that +does not appear in either completed row, and its wire definition must disappear. +Duplicate coverage includes all six targets and both other existing setters, +not just same-setter repetition. Used extension parameters must be checked on a +newly supported type too. Range diagnostics name the target type as well as its +raw code and retain the stable non-retryable reason without echoing input values. + +Early focused runs failed because the test oracle conflated two valid INT null +encodings. Source LONG_NULL and explicit string(null) produce a null bitmap; +a non-null LONG equal to INT_MIN narrows to four sentinel bytes with no bitmap. +Both read as SQL NULL, but the wire assertion must distinguish them. The +architect traced `schema-numeric-focused.2.4d/focused.log` to that fixture error +and required an input-aware assertion while preserving correct production +conversion. Negative DOUBLE IEEE +goldens also require unsigned hexadecimal parsing, not Long.decode. Include +the vector case ID in failures so these distinctions are directly diagnosable. +The server E2E uses its public cursor's effective-null semantics and is not a +substitute for these exact client wire assertions. + +Client final-source focused verification passed 20 tests; log: +`core/target/schema-numeric-focused-final.2.4/focused.log`. The earlier +`schema-numeric-focused.2.4` through `2.4e` logs retain the failed fixture +assertions and unsigned-bit parsing error described above. Those fixture +corrections did not change production conversion behavior. + +Final standalone client regression and installation: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-acceptance.2.4 -Dmaven.javadoc.skip=true install +``` + +PASS: **306 unit tests plus two packaging checks**, zero failures/errors/skips. +Log: `core/target/schema-numeric-acceptance.2.4/focused-install.log`. +The architect verified aggregate results and matching workspace/installed +`questdb-client-1.3.10-SNAPSHOT.jar` SHA-256: +`847c13d172067281cae5149604c2585eb3392ae0028be48429cd4a93bace29e4`. +The client author also verified byte identity with cmp. Installed tests-JAR +SHA-256: `955dda25657168fb0375edc6d4a09bcb98a6e45af35a6e0bb639f8c70e7286ce`. +Three additional serial runs used `-Dtest=QwpSchemaRowBufferTest`, `surefire:test` +and `-Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-repeats.2.4/run-`. +Each passed all 20 tests. Logs: +`core/target/schema-numeric-repeats.2.4/run-1.log`, `run-2.log`, `run-3.log`. +Independent review accepted final client and server source; real-server runtime +validation is still required before accepting iteration 2.4. + +## D012 — Hold acceptance: null-sentinel parity depends on legacy block contents + +Date: 2026-09-09. Status: source and real-server confirmed compatibility issue; +the hold was resolved by the user decision in D013 on 2026-09-10. The evidence +below records why the original parity assumption could not be accepted. + +The first server runtime run also exposed the INT_MIN bitmap/sentinel assertion +mistake. Inspecting why the public cursor reported a present value led the +architect to a deeper issue: QwpFixedWidthColumnCursor.advanceRow only invokes +isCurrentValueSentinelNull when there is no null bitmap. D011's statement that +source LONG_NULL is always classified before conversion was incomplete. + +Ordinary QwpWebSocketSender.longColumn creates a nullable LONG column but calls +addLong with the raw value. An omitted row in the same column/block causes a +bitmap to be emitted. A supplied LONG_MIN value with its bitmap bit clear is +then treated as a present number: narrowing to BYTE/SHORT/INT fails its range +check; FLOAT/DOUBLE casts produce a finite large negative value. Without that +bitmap, the same supplied bits are treated as null. The LONG-to-LONG same-width +path still stores the LONG null sentinel. This is block-context-dependent +legacy behavior, not an ordinary value-only converter. + +Independent review confirmed the chain through Sender, QwpTableBuffer, +QwpFixedWidthColumnCursor and WalColumnarRowAppender. The 47-vector legacy +corpus contains supplied values but no omitted row in the same column, so it +does not establish parity for this discriminator. Existing LONG setter Javadoc +does not promise sentinel normalization, and the design's intentional omission +change does not by itself authorize changing a supplied LONG_MIN value. + +The architect recommends a fixed schema-mode rule: source null sentinels mean +null before buffering, independently of other rows. Preserve legacy client and +server behavior. Do not add block-dependent conversion, send-time transformation +or a server behavior change merely to make this test agree. This is an explicit +compatibility exception to supplied-value parity and needs user agreement before +the design is revised or iteration 2.4 is accepted. Current client code already +implements the proposed rule but remains a dormant component; ordinary Sender +activation is still absent. The server author is adding a real-server diagnostic +with raw LONG_MIN and omission in the same block; no server production edit is +authorized by this finding. + +Runtime confirmation: the fresh server gate initially passed 268 tests (the +previous 265 plus three numeric E2E tests); log +`core/target/schema-numeric-server.2.4/full-regression.log`. This was before the +mixed-bitmap discriminator and is not final acceptance evidence. Three added +diagnostic tests in QwpSchemaLongNumericE2ETest now intentionally fail the +previous parity assumption: + +- BYTE receives an out-of-range SCHEMA_MISMATCH NACK for the block. +- FLOAT stores `-9.223372E18` for supplied LONG_MIN, followed by NULL for omission. +- DOUBLE stores `-9.223372036854776E18`, followed by NULL for omission. + +Log: server `core/target/schema-numeric-server.2.4/legacy-min-bitmap-diagnostic.log`. +The architect verified all three failures and their actual server outcomes. +The diagnostic methods are `testLegacyLongMinWithBitmapNullToByteMatchesSchemaConversion`, +`testLegacyLongMinWithBitmapNullToFloatMatchesSchemaConversion` and +`testLegacyLongMinWithBitmapNullToDoubleMatchesSchemaConversion`. They preserve +the failed assumption visibly; do not silently weaken them while claiming +parity. Source is frozen, server seeded repeats stopped, and no server production +changes were made. Once the user chooses the contract, update these tests to +assert the agreed legacy/schema distinction or the agreed parity rule and rerun +the complete acceptance gate. The client 306+2 gate and artifact remain valid +evidence for the proposed component, not a claim that the product contract is met. + +## D013 — Approved: consistent LONG null handling in schema mode + +Date: 2026-09-10. Status: user approved; accepted and verified in iteration 2.4. + +The user agreed that adding an empty row should not change another row's value +and accepted the recommendation that the new client should not reproduce the +server inconsistency. Adopt the previously proposed rule: for supported numeric +targets, LONG_MIN is source null before buffering, independent of block contents. +Legacy behavior remains unchanged. This approval is not authorization for a +separate legacy server fix or ordinary Sender activation. + +Design revision 15 records the rule as an explicit exception to supplied-value +conversion parity. Target missing representation remains zero for BYTE/SHORT +and NULL for INT/LONG/FLOAT/DOUBLE. Duplicate-first behavior and input/target +support validation precede normalization; LONG_NULL must not enable an otherwise +unsupported pair. The already-reviewed client component implements this rule; +no additional production change is needed to resolve the hold. + +Replace the three failed parity assumptions with deliberate compatibility tests. +They must separately prove the preserved legacy outcomes and the stable schema +outcomes with and without omission in the same column/block. Include the full +six-target numeric family, exact wire representations and independent SQL +expectations. Preserve the failed diagnostic log as evidence of the decision; +do not describe the resulting green tests as full legacy parity. + +SOL server author owns those test changes and the fresh real-server gate; +SOL client author revalidates the unchanged artifact and client gate; independent +SOL reviewer challenges the contract, source and assertions. The architect owns +the design, journal and final acceptance. Keep per-repository Maven runs +serialized, use the installed standalone client, and preserve unrelated work. +No server production changes, send-time transformation, extra mode switch or +per-row allocation mechanism are introduced. A legacy-server bug fix remains +separate work with its own compatibility review. + +## Iteration 2.4 acceptance — approved null contract + +Date: 2026-09-10. Status: accepted. The D012 hold is closed by D013 and the +fresh checks below, not by claiming full legacy conversion parity. + +The final server test replaces the three false-parity diagnostics with +`testLongMinSchemaNullIsStableWhileLegacyDependsOnBlockBitmap`. It covers all +six numeric targets, each with and without an omitted row, separately for +legacy and schema encoding. Wire assertions retain legacy TYPE_LONG and raw +LONG_MIN bits, prove when a bitmap exists and whether the value is present, +and contrast schema encoding's target type and explicit missing-value bit. +SQL assertions independently verify stable schema missing values and the +preserved legacy range errors or finite FLOAT/DOUBLE values. The existing +47-vector corpus, partial-row rollback, duplicate-first behavior and explicit +null/omission checks remain in the gate. The failed D012 log is retained. + +Fresh client verification, from the standalone client repository: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-final-verify.2.4 -Dmaven.javadoc.skip=true verify +``` + +PASS: **306 selected unit tests plus two packaging checks**, zero +failures/errors/skips. Log: +`core/target/schema-numeric-final-verify.2.4/focused-verify.log`. +No client source edits or reinstall were needed for this decision. + +Fresh real-server gate, from the server repository, without `local-client`: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaIdentityParserTest,QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaLongNumericE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide,QwpUdpMalformedTest,QwpUdpInsertTest' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-numeric-server.2.4 test +``` + +PASS: **269 selected tests**, zero failures/errors/skips. Log: +`core/target/schema-numeric-server.2.4/final-regression.log`. +The same command with only `-Dtest=QwpSchemaLongNumericE2ETest` first passed +all four numeric tests; log: +`core/target/schema-numeric-server.2.4/contract-focused-wire.log`. + +Three additional serial server runs used +`-Dtest=QwpSchemaLongNumericE2ETest,QwpSchemaRowBufferE2ETest`, `surefire:test`, +the same absolute temporary directory, and seed pairs supplied through +`-Dfuzz.s0`/`-Dfuzz.s1`: `(1,2)`, `(42,43)`, `(1234567,7654321)`. +Each passed **eight tests**, zero failures/errors/skips. Logs under +`core/target/schema-numeric-server.2.4/repeats/`: +`repeat-1-2.log`, `repeat-42-43.log`, `repeat-1234567-7654321.log`. + +The architect rechecked aggregate logs, final wire assertions and the Surefire +XML classpath, which uses the installed standalone client from +`/home/jara/.m2/repository/org/questdb/questdb-client/1.3.10-SNAPSHOT/`. +Workspace and installed main JARs retain matching SHA-256 +`847c13d172067281cae5149604c2585eb3392ae0028be48429cd4a93bace29e4`. +Both repositories' shared corpus retains SHA-256 +`0b574bc3b53fb7cc027b0cd2a9fc8d63f75a99d35f6f7f206d7bd21a70334f64`. +Independent SOL review accepted revision 15, final source and runtime evidence +with no remaining blocker. Both repositories pass `git diff --check`. + +Reflection: a corpus of individual supplied values was insufficient to prove +legacy parity because missing values elsewhere in a block change decoding. +Future conversion slices must vary omission/explicit-null context as well as +individual boundary values. Keep exceptions explicit in the contract instead +of letting batching determine conversion results. + +This completes the bounded LONG-input numeric component iteration. Ordinary +Sender setters remain unactivated; no legacy server production behavior, +send-time transformation or mode switch changed. No commits were made. +The next conversion family remains separate work requiring its own bounded +brief, source validation and real-server conformance tests. + +## D014 — Record integration simplification opportunities + +Date: 2026-09-10. Status: recorded at the user's request; implementation unchanged. + +Design revision 16 adds a separate "Simplification opportunities" section after +the progress reflection: reuse Sender-owned row storage and lifecycle, keep +lookups with the existing I/O owner, share schema metadata/payload machinery, +and make current implementation status easy to find. These are integration +guidelines, not completed refactors or authorization for partial activation. +Pinned snapshots, compatibility, bounded requests and delivery rules remain +unchanged. The implementation-status checklist is recorded as an opportunity; +this documentation change does not add it or select the next implementation +slice. + +## D015 — Schema feedback before further conversion families + +Date: 2026-09-10. Status: accepted. The fresh SOL review reopened the initial +acceptance; D016 records the correction, independent review and final regression +evidence that restored it. Wire contract and scope remain unchanged. + +Following the progress reflection, the user asked to continue. Prioritize the +schema-update path over another isolated converter family. This bounded slice +adds real server ACK/NACK feedback and a production client decoder, exercised +with the existing conversion component. Ordinary Sender activation, latest- +schema cache ownership/adoption, lookup coordination and automatic row-boundary +transitions remain future work. Do not add an unused callback/cache framework +or activate only selected setters to disguise those gaps. + +Root and three SOL agents inspected response parsing, metadata discovery, +cumulative ACKs, deferred commit groups, partial sends and schema-required SF +replay before freezing the contract. Current table identities are parsed and +persisted but are not consumed by ingestion; this slice gives them a feedback +purpose, never a version precondition. Negotiated legacy blocks also have an +unknown identity and can receive feedback. + +Revision 17 defines status mode bits: `0x00` unchanged response, `0x80` complete +update list, `0xC0` invalidate all future lookup-cache entries, `0x40` reserved. +Low six bits retain the ordinary status. Feedback requires negotiation; +durable ACKs never carry it. Named, length-delimited entries reuse the SCHEMA +payload with request ID zero reserved for write feedback. DESCRIBE remains +positive-ID. Full validation precedes ACK/NACK handling and metadata exposure. + +Simplification chosen after adversarial review: if the complete metadata set +cannot fit the 1 MiB UPDATES-payload and raw-response-buffer bounds, send +invalidation instead of paginating or repeating cumulative ACKs. Encoding the +mode in existing status bits makes invalidation exactly the base response's +length. This avoids even a one-byte capacity regression and requires no new +send continuation. It affects future lookup cache entries only, never pinned +rows, persisted bytes, delivery watermarks or terminal-error policy. + +Collect bounded names/identities; obtain coherent, freshly authorized snapshots +when serializing responses. Keep successful ACK-prefix feedback separate from +rejected-frame/deferred-group feedback. Preserve both across backpressure under +the existing response ownership; clear only the fully sent set. NACK metadata +may cover authorized tables encountered in rejected work, not unrelated tables +or metadata owned by a pending earlier ACK. Unavailable/unauthorized complete +feedback may use nameless invalidation without changing the data outcome. + +Assignments: + +- SOL server author: scoped server metadata/feedback state and wire writes, + real-server feedback/evolution, authorization, bounds, ACK ordering and + partial-send tests. No WAL conversion or UDP behavior change. +- SOL client author: shared schema payload codec, negotiation-aware response + decoding, existing foreground/background replay consumers, malformed-response + and delivery-regression tests. No alternate Sender API or unused observer. +- Independent SOL reviewer: adversarial wire, ownership, bounds, compatibility + and final source/evidence review, read-only and separate from authors. +- Architect: design, journal, scope, source assumptions and final acceptance. + +Per-repository builds remain serialized. Use the installed standalone client, +never the server's `local-client` profile, and verify artifact identity. Preserve +all existing dirty work and the unrelated server Rust checkout. This is +functional protocol work, not a performance optimization: no bottleneck or +speedup claim, speculative tuning or new per-row conversion work is authorized. +Performance remains unmeasured and must not be inferred from test time; +performance optimization would require the hardware-first measurement gate. + +Pre-acceptance clarification: root challenged applying the 1 MiB limit to +INVALIDATE_ALL. An existing base ACK can exceed 1 MiB when the configured +response buffer and table set are large. The new limit therefore applies to +the complete UPDATES payload only; NONE and INVALIDATE_ALL keep existing base +bounds. Independent review agreed. Add paired codec checks for a large valid +base ACK and its same-length invalidation form, while rejecting oversized +UPDATES. This preserves the fallback's zero-added-size guarantee without +introducing a global receive cap. + +## Iteration 2.5 development evidence + +Source baselines remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and +server `12a33d651e51e2682e7a448c8db5168fc72dfad3`, plus the preserved dirty +changes from accepted iterations. Root verified both HEADs and the installed +artifact used below. All builds use repository-local temporary storage because +the shared `/tmp` has little space; unrelated files are not removed. + +Client pre-edit response/replay/durable baseline passed 41 tests: +`core/target/schema-feedback-baseline.2.5/baseline.log`. The first focused run +failed the old unknown-status fixture using `0xFF`, which is now an explicitly +negotiated response mode rather than an unqualified legacy error. The fixture +now uses unknown base status `0x3F`; separate tests reject unnegotiated modes. +This is a protocol-reservation correction, not removal of unknown-base-status +coverage. Failed log: `core/target/schema-feedback-focused.2.5/focused.log`. +The subsequent focused gate passed 58 tests: +`core/target/schema-feedback-focused3.2.5/focused.log`. + +The initial standalone install passed 331 selected tests plus two packaging +checks; log `core/target/schema-feedback-install.2.5/install.log`. Workspace +and installed main JARs match SHA-256 +`e9e9d93f63a1a9a25b33163a6db4a027152bb8c174030b965450bb64211e6825`. +This supplies the server with the production decoder. Network tests and final +source validation were still being extended afterward; this is not iteration +acceptance evidence by itself. + +Construction review found and required corrections before acceptance: + +- A reset block had landed in `onAckBlocked`, clearing rejected/deferred + metadata before copying ACK ownership. Keep connection reset separate from + the parked ACK snapshot; test a blocked ACK followed by a distinct NACK. +- Table-writer acquisition can fail before append, for example for a non-WAL + table. Capture encountered-table feedback on that path too, without masking + the original failure or adding metadata reads to matching successful writes. +- Preflight declared update count against available minimum entry bytes before + allocation; validate table names with server-compatible case-insensitive + duplicate checks and existing name rules. +- A size-limit rejection test must contain an otherwise complete valid UPDATES + response; a missing trailer would reject independently of the limit. Use + valid, distinct table names in the large base-response compatibility fixture. +- WebSocket short-payload capacity must clamp at 125 bytes for 128/129-byte + buffers, before the larger header becomes necessary. + +These are in-progress review findings, not defects in an accepted iteration. +Final runtime gates and independent stable-source review remain required. + +Client canonical final-source verification passed 333 selected tests plus two +packaging checks; log `core/target/schema-feedback-canonical.2.5/verify.log`. +Three serial runs of `QwpSchemaFeedbackResponseTest,QwpSchemaReplayNetworkTest` +each passed 15 tests; logs +`core/target/schema-feedback-repeats.2.5/run-{1,2,3}/test.log`. +The production artifact hash remains `e9e9d93f...`; no reinstall occurred while +the server used it. Independent client review accepted source and evidence. +The malformed-response network test uses an observable second replay attempt +and bounded close configuration, then reopens public SF state to verify +`ackedFsn=-1`, retained published frames and no quarantine. Earlier unbounded +test-harness attempts are not counted as successful validation. + +Initial real-server checks passed five identity E2Es and two new feedback E2Es; +logs `core/target/schema-feedback-server.2.5/identity-test.log` and +`feedback-e2e.log`. The exact-capacity test sends a 252-byte base ACK plus its +four-byte WebSocket header through a 256-byte response buffer, preserving both +table transaction entries and SQL rows with same-length invalidation. The +pre-writer non-WAL failure returns authorized current metadata and no rows. +These are partial gates, not final server acceptance. + +Independent server review also found that resumed NACK completion did not +release its metadata ownership like immediate completion; this was corrected. +Root challenged a proposed later-unrelated-NACK test: the existing unresolved +sequence guard prevents that ordinary data sequence on the same connection. +The reviewer agreed to test reachable partial-NACK completion and retained +delivery behavior, not invent a private-state test or claim a demonstrated leak. + +After the client gate completed, the client SOL author took ownership of the +new server `QwpSchemaFeedbackAuthorizationE2ETest.java` only, in parallel with +the server author's remaining lifecycle tests. Server production and Maven +remain solely server-author-owned; the independent reviewer remains read-only. + +The client author subsequently added a separate evolution E2E and handed both +test files to the server author before combined compilation. Initial combined +runs exposed fixture errors: invalid WAL DDL, duplicate frame-handler methods, +using a wire type constant for a metadata type, and an old response helper that +did not pass negotiated capability. These failures are preserved in the +`feedback-matrix-*.log` files, not counted as successful gates. Root also +required ordered DESCRIBE barriers before revocation or ALTER, rather than +assuming that sending DATA proves server processing. Native encoder pointers +must be read after encoding, which can grow the underlying buffer. + +Revocation tests corrected a load-bearing assumption: the existing commit path +checks INSERT authorization again. Revoking it after deferred ingestion can +produce a SECURITY_ERROR NACK, not a successful ACK. Test the existing data +outcome and nameless metadata invalidation; do not change authorization to make +the original fixture expectation pass. + +Root challenged the claim that the existing live-table limit also bounded +pending feedback. A proposed rowless deferred/drop/eviction sequence appeared +to allow historical names to accumulate, and root temporarily authorized an +explicit bounded accumulator with an invalidation marker. Both author and +reviewer initially accepted the counterexample. Further source tracing then +disproved its premise on the WebSocket path: the independent stale-entry scan +is UDP-only. WebSocket lookup evicts under the same requested name, while a new +distinct name hits the table ceiling; rename and buffered-drop eviction paths +reject the frame. The captured table token also does not silently follow pure +renames. No unbounded WebSocket sequence was established. + +Correction and final decision: withdraw that blocker and the proposed new +accumulator. Restore the simpler direct maps, bounded by the configured table +ceiling plus at most the terminal failing name. Document this source dependency +and revisit it if cache eviction changes. Do not add an invalidation latch or +new test obligations merely to defend the earlier hypothesis. The temporary +helper compiled but is not part of the accepted change. This corrects root's +earlier user-facing claim of a reachable gap as well. + +One independently verified simplification remains: remove the redundant parked +ACK metadata copy. The existing raw send buffer already owns the exact serialized +bytes; resuming that send needs no retained or resampled schema objects. Keep +pending rejected-work metadata separate until its NACK is serialized. Final +server runtime and review acceptance remain pending. + +The corrected server component matrix passed 26 tests in +`core/target/schema-feedback-server.2.5/feedback-matrix-7.log`. In particular, +the mixed authorization test preserves the existing partial-commit outcome: +the earlier authorized table retains its row, the denied table has none, and +the NACK exposes no named schema. This is not an all-or-nothing transaction +guarantee. The evolution E2E verifies INT bytes encoded before ALTER, LONG +metadata in the stale write's ACK, byte-identical re-encoding from the old +component, and a subsequent LONG wire value above INT_MAX with no redundant +feedback. The combined gate still precedes final cleanup and broad regression. + +Canonical client verification command (client repository; selected tests, not +the entire project suite): + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,QwpSchemaFeedbackResponseTest,WebSocketResponseTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-feedback-canonical.2.5 -Dmaven.javadoc.skip=true verify +``` + +Each client repeat uses the same command shape with +`-Dtest=QwpSchemaFeedbackResponseTest,QwpSchemaReplayNetworkTest`, `test` rather +than `verify`, and its own absolute temporary directory under +`core/target/schema-feedback-repeats.2.5/run-{1,2,3}`. + +The first broad server gate ran 280 tests with two failures, both in existing +LONG-numeric E2Es whose response helper negotiated schema but still invoked +the legacy decoder overload. Negotiated legacy table blocks correctly return +feedback for their unknown identity. Correct the test helper to pass connection +capability; do not suppress production feedback or weaken SQL/wire assertions. +Failed gate preserved as `core/target/schema-feedback-server.2.5/final-canonical.log`. +The other 278 tests, including all new feedback cases, passed; the complete +corrected gate and seeded repetitions are still required. + +## Iteration 2.5 final acceptance — 2026-09-10 + +Accepted by the architect after the independent SOL review returned ACCEPT and +both authors froze their source. No remaining source or targeted-runtime blocker. +This accepts the bounded feedback slice, not automatic schema-mode Sender. + +- Client canonical gate: **333 selected tests plus two packaging checks**, zero + failures/errors/skips. The seven network replay tests are included in 333, + not an additional count. Three serial feedback/replay repeats passed **15 + tests each**. Logs and command are recorded above. +- Server canonical gate: **280 selected tests**, zero failures/errors/skips. + Log: `core/target/schema-feedback-server.2.5/final-canonical-2.log`. +- Server focused repeats: **19 tests each**, zero failures/errors/skips, with + seeds `(1,2)`, `(42,43)` and `(1234567,7654321)`. Logs: + `core/target/schema-feedback-server.2.5/repeats/repeat-1-2.log`, + `repeat-42-43.log` and `repeat-1234567-7654321.log`. +- Root reverified the matching workspace/installed client JAR SHA-256 + `e9e9d93f63a1a9a25b33163a6db4a027152bb8c174030b965450bb64211e6825`. + Surefire XML points to the installed standalone `.m2` artifact, not + `local-client`, and confirms the final seed pair and repository-local temp + directory. Both repositories pass `git diff --check`. + +Canonical server command (server repository): + +```sh +mvn -B -pl core -DargLine='-ea -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-feedback-server.2.5/tmp -XX:+UseParallelGC --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.time.zone=ALL-UNNAMED --add-exports=java.base/jdk.internal.vm=ALL-UNNAMED --add-exports=java.base/jdk.internal.vm=io.questdb' '-Dtest=QwpSchemaIdentityParserTest,QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaLongNumericE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpSchemaFeedbackE2ETest,QwpSchemaFeedbackAuthorizationE2ETest,QwpSchemaFeedbackEvolutionE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide,QwpUdpMalformedTest,QwpUdpInsertTest' test +``` + +Server repetitions reuse the identical `-DargLine`, replace the selection with +`-Dtest=QwpSchemaFeedbackE2ETest,QwpSchemaFeedbackAuthorizationE2ETest,QwpSchemaFeedbackEvolutionE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaLongNumericE2ETest`, +use `surefire:test`, and add `-Dfuzz.s0=1 -Dfuzz.s1=2`, then `42/43`, then +`1234567/7654321`, serially. Passing `java.io.tmpdir` in the fork's JVM arguments +is important: earlier server runs still used `/tmp` despite a Maven-level +property. Final XML verifies the intended location. No unrelated temporary +files were deleted. + +Delivered behavior is demonstrated through real negotiated writes and observable +wire/SQL outcomes: matching identities omit feedback; stale/unknown identities +return current authorized metadata; oversized or denied complete feedback uses +nameless invalidation; deferred groups coalesce names; describes and subsequent +ACKs agree after ALTER; partial sends preserve separate ACK/NACK contents; old +INT bytes remain unchanged while a later component encodes LONG. Malformed +feedback cannot release persisted replay data. Metadata does not change ordinary +or durable ACK coverage, rejection policy, or partial-commit semantics. + +Simplicity decisions retained: shared schema payload codec, one complete update +set or same-length invalidation, response-time snapshots, and no duplicate +metadata owner for an already serialized ACK. The speculative accumulator was +removed, and its disproved motivating sequence is recorded above rather than +presented as a fixed defect. + +Remaining work: latest-schema cache ownership/adoption, lookup coordination, +automatic row-boundary transitions, the remaining conversion families and +coherent activation in ordinary Sender. The next architecture slice should +resolve producer-owned row/block integration using the existing buffer owner; +do not grow another independent row store or activate a subset of setters. +Tests are a targeted regression matrix, not exhaustive proof of every existing +close/durable permutation. Performance is unmeasured; no throughput or latency +claim follows from these functional checks. No commits or pushes were made; +unrelated work, including the server Rust checkout, was preserved. + +## Fresh independent SOL review — 2026-09-10 + +At the user's request, a new SOL agent reviewed iteration 2.5 independently, +without the implementation conversation. It inspected the design, current +client/server source, tests and recorded execution evidence. Root separately +reverified artifact identity, test counts, seeds and temporary paths. No Maven +rerun, implementation edit or commit was performed during this review. + +Verdict: reopen iteration 2.5 acceptance for one **P2** finding. Root agrees. +`QwpSchemaProtocol.decodePayload` checks column-name length and UTF-8 but does +not apply column-name rules or reject case-insensitive duplicates. A schema +containing `x` and `X`, or an illegal name such as `a/b`, can therefore pass the +shared decoder. `WebSocketResponse.readFeedback` trusts that decoded schema; +the successful ACK path can then advance replay progress. Duplicate detection +in `QwpSchemaRowBuffer` occurs later and does not protect the ACK boundary. + +The reviewer reproduced duplicate-column acceptance against the verified +installed client JAR: `decoded=2 names=x,X`. Acceptance of the enclosing ACK and +the replay-watermark consequence were traced in source, not reproduced through +a network replay. No data-loss incident or malformed response from the current +server was demonstrated. The issue is the promised rejection of malformed +metadata before delivery progress; existing replay tests cover malformed +framing, not these invalid column definitions. + +Recommended correction, not implemented in this review: validate server-compatible +column names and case-insensitive uniqueness in the shared schema decoder. Add +DESCRIBE and ACK-feedback rejection tests plus a public replay-network case +showing the malformed schema leaves `ackedFsn == -1`. Rerun the affected gates +before restoring acceptance. + +No other blocking defect was found. A suspected combined-response 1 MiB sizing +mismatch was disproved: the caller subtracts the base response from the capped +complete-payload budget before encoding feedback. Missing-table create-permission +reporting was not promoted to a defect: MISSING reports absence, not permission +to create, and actual creation still enforces authorization. Automatic cache +adoption and ordinary Sender row transitions remain outside the reviewed slice. + +## D016 — Validate schema column names at the decoding boundary + +Date: 2026-09-10. Status: accepted and verified; iteration 2.5 acceptance restored. +The user asked to fix the P2 without extra complexity. + +Put column-name validity and case-insensitive uniqueness checks in the existing +shared decoder loop. Reuse `TableUtils` and an existing bounded hash collection; +do not add a validator framework, configuration, wire field, lookup state or +quadratic scan. Both DESCRIBE and write feedback then share the same rejection +boundary. This is schema-metadata work, not per-row conversion validation. + +Root verified that `QwpSchemaResponse` is constructed only by the decoder and +its constructor is package-private. Once decoding guarantees valid unique names, +remove the later duplicate-error branch from `QwpSchemaRowBuffer`; retain the +name-to-column binding map it actually needs. Do not retain another name index +in the immutable schema just to avoid a short-lived decoder collection. + +The new SOL author owns the two scoped client production files and existing +codec/feedback/replay tests. The fresh SOL reviewer remains independent and +read-only. Root owns architecture, journal and acceptance. Preserve all earlier +dirty work and failed-run evidence; no commits, ordinary Sender activation, +other converters or server implementation changes are part of this fix. + +Require failing tests on the old decoder, then a green focused gate: direct +DESCRIBE and ACK/NACK feedback reject duplicate/illegal column names before +publishing any metadata. Preserve valid Unicode, spaces and boundary-length +names. Reuse the public replay-network harness to show semantically malformed +feedback retains the published backlog with `ackedFsn == -1`, rather than just +asserting a parser exception. No reflection, private-state hooks or fixed sleeps. +Revalidate the client aggregate and real-server matrix with the newly installed +standalone artifact before restoring iteration 2.5 acceptance. Tests remain +functional evidence; no performance claim is made. + +D016 construction evidence: the initial focused red run failed the new direct +DESCRIBE and feedback checks, but the new network retention test still passed +with the old decoder. Root traced this to the existing feedback test helper: +its latch counted two replies, while the seeded backlog contains two frames. +Both replies could be sent on the first connection, followed by immediate +sender close before either ACK was processed. The earlier description of this +as an observable second replay attempt was therefore too strong. + +Require replies on distinct peer connections before closing the sender. This +observes an actual reconnect without production hooks, and strengthens both +the existing truncated-feedback case and the new invalid-column case. Verify +that the corrected semantic network test fails against the previous installed +artifact, without restoring production source to manufacture the red run. +This is test coordination, not additional production state. + +The first post-fix focused run also failed native library extraction because +the configured JVM temporary directory did not exist (`File.createTempFile` +reported ENOENT). Creating the scoped directory corrected the environment; +no native or production-code change was required. Preserve that failed log +alongside the rerun; it is not acceptance evidence. + +The first completed aggregate run found one stale row-buffer test expecting the +later duplicate-name exception. Remove that redundant test: the decoder tests +now own this invariant, and invalid schemas cannot reach the row buffer. +This changes the final selected client count from 337 to 336 without dropping +coverage. An earlier incomplete aggregate log is also retained and is not +counted as a passing gate. + +### D016 final verification and acceptance + +The production change is confined to the shared schema decoder and removal of +the redundant row-buffer check. The decoder reuses the existing +`LowerCaseCharSequenceIntHashMap`, bounded by the existing 2,048-column limit. +Its case comparison and `TableUtils` name rules match the server. There is no +new production type, protocol field, setting or retained state. The row buffer +still has its required binding map. Server source is unchanged by D016. + +The final public-Sender network test fails against the previous installed client +JAR, SHA-256 +`e9e9d93f63a1a9a25b33163a6db4a027152bb8c174030b965450bb64211e6825`. +The old JAR precedes compiled test classes on the classpath. It accepts the +malformed feedback and stays on one connection; the test fails specifically at +`invalid schema feedback must be rejected and replayed`, not native startup. +With the fix, two distinct peer connections are observed and reopening the +published backlog verifies `ackedFsn == -1`, `publishedFsn == 1` and no quarantine. +The direct codec and whole-ACK/NACK tests also reject illegal names and exact, +ASCII-case and Unicode-case duplicates, while retaining valid name boundaries. + +Final evidence (all successful gates have zero failures, errors and skips): + +- Client: **336 selected tests and two packaging checks** in + `core/target/schema-feedback-name-fix/canonical-pass.log`. +- Client feedback/replay repeats: **17 tests each, three runs**, in the same + directory's `repeat-1.log`, `repeat-2.log` and `repeat-3.log`. +- Old-artifact negative proof: `final-network-old-installed-red.log`; initial + focused codec/feedback negative proof: `focused-red.log`. Failed environment + runs and the stale-test aggregate failure remain separate from acceptance. +- Server: **280 selected tests** in + `core/target/schema-feedback-name-fix-server/canonical.log`. +- Server feedback/conversion repeats: **19 tests each**, seeds `1/2`, `42/43` + and `1234567/7654321`, in that directory's corresponding `repeat-*.log` files. + +The installed standalone client and workspace JAR have identical SHA-256: +`9eb4c6716a3dd65c8c4bf6e2de92427f80a7c7a2dc4840aadfbdb75322064fa0`. +Root verified this before and after the server runs, and verified the installed +JAR on the server test classpath. The server did not use `-Plocal-client`. +Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD remains +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with the ongoing task's uncommitted +changes. Builds were serialized per repository. Test JVMs used existing, +absolute repository-local temporary directories, including forked JVM options. + +Exact commands, including the old-artifact classpath reference and per-run JVM +temporary paths, are recorded in the local +[run manifest](../core/target/schema-feedback-name-fix/run-manifest.md). Client +and server test selections match D015; the client aggregate includes the new +tests and removal of the redundant row-buffer test described above. + +The independent SOL reviewer accepted the final source, client and server +evidence. Root independently verified the gates and artifact provenance before +restoring acceptance. Both repositories pass `git diff --check`. This closes +the P2, not the remaining latest-schema adoption or ordinary Sender activation +work. No commits, pushes or performance claims were made. + +## D017 — Separate schema conversion from buffer ownership + +Date: 2026-09-10. Status: accepted and verified; iteration 2.6. + +The user approved the next integration step. Root's preceding recommendation +overstated how soon automatic INT-to-LONG adoption could be exercised through +ordinary Sender. Read-only SOL review and root's source trace found no +production consumer for decoded schema updates, no first-use DESCRIBE +coordinator, and no complete Sender conversion/inference integration. Starting +with a legacy probe and adopting feedback would bypass required first-use +lookup and constitute the prohibited partial activation. Do not manufacture +that E2E by adding an opt-in or test-only Sender mode. + +Deliver the ownership prerequisite first: make the conversion component bind +to an existing `QwpTableBuffer`, without allocating, closing, resetting, +completing or encoding a second row store. Keep one owner for buffer lifetime, +row completion, cancellation, byte/row accounting and encoding. Preserve the +existing UUID and LONG numeric conversion contracts and real-server coverage. +No borrowing/owning flag, alternate buffer queue, cache scaffold or ordinary +Sender activation belongs in this increment. Pin schema identity to the bytes +at the owner boundary; a borrowed converter must not provide a way to mix two +snapshots in a non-empty buffer. + +Before editing, root read the hardware-first-performance skill and its required +USE/hardware references; SOL performed a read-only capability preflight. There +is no existing benchmark for the schema component's fixed-work write/encode +path. The available Sender latency benchmark exercises a different, non-schema +path, and the host's current system-wide I/O pressure needs workload-correlated +analysis before any resource classification. The skill's optimization study +cannot be applied cleanly to this bounded functional task without creating a +separate measurement project. Fallback: no performance optimization or claimed +bottleneck/speedup, no change to ordinary Sender's row setters, source review of +ownership/allocation changes, and functional regression evidence only. Encoding +entry points gain guards for newly schema-bound buffers; do not describe that +as zero code change on the existing encoding path or as measured zero overhead. + +Root owns this brief, documentation and acceptance; SOL implementation and an +independent SOL review remain separate. Preserve all prior dirty work, serialize +builds per repository, use existing absolute repository-local JVM temp paths, +install the standalone client before real-server validation, and never use +the server's `-Plocal-client` profile. No commits or pushes are authorized. + +D017 API decision: replace the unreleased owning `QwpSchemaRowBuffer` with +`QwpSchemaBinding(QwpTableBuffer, QwpSchemaResponse)`. The owner creates and +closes the table buffer, completes rows, and cancels/rolls back a partial row +when a setter fails. The binding only resolves, converts and appends values. +Do not retain a compatibility wrapper that owns another buffer lifecycle. + +The table buffer holds its current binding reference. Attach only to a wholly +empty, unbound layout, after validating/constructing the binding. `reset()` +preserves the attachment and column definitions; `clear()` detaches before +cleanup. Each binding setter checks that it is still current before touching +the buffer. Root rejected the proposed separate layout-lease object and +generation counter: reference identity already detects stale bindings, and the +binding already holds the immutable schema. Do not duplicate schema identities +or introduce a reattachment API for previously invalidated bindings. + +Schema encoding derives identity from the buffer's binding. Existing explicit +identity entry points remain valid for unbound protocol buffers and bound +buffers with the exact pinned identity; reject mismatches before changing the +encoder. Legacy encoding rejects a bound buffer rather than stripping its pin. +For bound buffers, partial rows cannot be encoded; an empty single-table encode +returns zero without modifying the encoder, while appending an empty bound table +to a predeclared multi-table frame is rejected. Existing unbound-buffer encoding +contracts remain unchanged. These are owner-misuse errors, not value-conversion +failures or reasons to cancel another row automatically. + +Real-server tests must use this explicit owner lifecycle and retain their SQL, +wire and shared-vector assertions. Evolve a caller-owned buffer from INT to +LONG only after retaining its old encoded bytes, clearing its old layout and +binding the new schema. This proves the ownership prerequisite and immutable +old bytes; it is explicitly not automatic feedback adoption by Sender. + +### Iteration 2.6 development and review evidence + +The first migrated client gate ran 19 tests and found three callers still +assuming the old facade's automatic rollback. The replacement binding correctly +left cancellation to the owner; the tests needed explicit owner cancellation +and uncommitted-column rollback. Keep their original row/value expectations, +and rename the key regression to describe owner rollback. Preserve the failed +run as `core/target/schema-binding-2.6/logs/focused.log`, not acceptance evidence. +The completed focused suite has 23 tests, including reference-based stale +binding rejection for all three setters while a newer row remains intact, +same-snapshot clear/rebind, reset/layout retention, failed-constructor reuse, +and byte-level encoder checks for bound-buffer framing, identity and partial/ +empty-row errors. + +The independent SOL reviewer accepted the reference-identity attachment and +verified the final guard ordering. Root required rejection before the +`encodeWithDeltaDict` convenience method starts a new frame, and preserved +matching explicit-identity encoding as byte-identical to derived encoding. +No lease object, counter, additional schema representation or owning adapter +was needed. Root also required explicit owner cleanup in failed-constructor +tests and verified that the same buffer accepts a valid binding afterward. + +Three existing real-server E2E classes were migrated without changing server +production code. The first build stopped at test compilation because a renamed +buffer local shadowed a table-cursor local; no tests ran. The corrected focused +gate passed nine tests. Root restored the pre-existing old-frame re-encoding +equality assertion before clear/rebind and strengthened wire assertions to +check exact old/new table identities, not merely non-negative identities. +SQL, UUID/numeric corpus, null semantics and completed-row preservation checks +remain intact. These tests still exercise the production components over a real +connection; they do not pretend to exercise automatic ordinary Sender adoption. + +An incremental-build rename can leave obsolete classes in the artifact. Root +required source, compiled-output and JAR checks for the removed owning facade. +No stale compiled files were found, so no cleanup or quarantine was needed. +Earlier logs were preserved, and neither repository was blanket-cleaned. + +Design revision 18 adds a short current implementation-status section, as +suggested by D014, so the intended contract cannot be mistaken for a completed +feature. Detailed history and test evidence remain here. + +### Iteration 2.6 final acceptance — 2026-09-10 + +Accepted by root following independent SOL source/adversarial review and +completed client/server gates. This accepts the non-owning binding prerequisite, +not automatic schema adoption or normal Sender activation. + +- Client final canonical gate: **340 selected tests plus two packaging checks**, + zero failures/errors/skips. The 23 binding tests are included in 340. Log: + `core/target/schema-binding-2.6/logs/canonical-final.log`. +- Server focused migrated E2Es: **9 tests**, then canonical **280 tests**, all + green. Logs: `core/target/schema-binding.2.6-server/focused3-rerun.log` and + `canonical280.log` in the server repository. +- Three server feedback/conversion repeats: **19 tests each**, seeds `1/2`, + `42/43`, `1234567/7654321`, zero failures/errors/skips. Logs in the same server + directory: `repeat-1-2.log`, `repeat-42-43.log`, + `repeat-1234567-7654321.log`. + +Client canonical command, run from the client repository: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,QwpSchemaFeedbackResponseTest,WebSocketResponseTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaBindingTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-binding-2.6/tmp -Dmaven.javadoc.skip=true verify +``` + +The focused client command selects `QwpSchemaBindingTest` and uses `test` with +the same temporary path; install used +`mvn -B -pl core -DskipTests -Dmaven.javadoc.skip=true install` after the first +green canonical gate. Final test-only constructor assertions were subsequently +verified in `canonical-final.log`, without changing or reinstalling the main +production artifact. Client JVM reports confirm the repository-local temporary +directory; the command relies on existing POM fork options and does not add a +separate temporary-directory option to `argLine`. + +Server selectors, JVM flags, temporary paths and seeded commands are recorded +in the local [server run manifest](/home/jara/devel/oss/questdb/core/target/schema-binding.2.6-server/run-manifest.md). +The installed standalone client was used, never `-Plocal-client`; root verified +the test classpath, final seed/temporary path, and before/after artifact hashes. +The workspace and installed main JAR share SHA-256: +`f14a8793c882bcbf5515c911376c823d8addd17e7013ed9d6f35858173b3b3d0`. +Both contain `QwpSchemaBinding` and no `QwpSchemaRowBuffer` class. + +HEADs remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with ongoing uncommitted task changes. +Both repositories pass `git diff --check`. Only the three scoped server test +files changed in this increment; no server production code, commits or pushes. +Functional tests do not establish a performance improvement or zero overhead. + +Next dependency: a real first-use DESCRIBE/feedback handoff and latest-schema +cache, with connection invalidation, bounded waits and producer-only adoption. +Do not add a feedback-only legacy probe or enable a subset of ordinary Sender +setters. The remaining conversion/inference contract and owner integration must +be coherent before automatic schema mode is activated. + +## D018 — Iteration 2.7: one lookup handoff and one latest-schema cache + +Date: 2026-09-10. Status: accepted; final verification below. + +The next functional slice connects existing schema codecs and ACK/NACK feedback +to the real cursor I/O loop. Root owns architecture, decisions and acceptance; +SOL owns client implementation, real-server tests and independent/adversarial +review. Ordinary Sender activation and row-boundary adoption remain out of +scope. This must produce an observable real-loop E2E, not another unused codec +or a feedback-only legacy probe. + +Decisions and source-checked simplifications: + +- One producer needs one outstanding lookup, not a general request queue. A + coordinator monitor protects the request and a lazy insertion-order cache + capped at 1,000,000 entries. Do not combine a concurrent map, eviction queue, + atomic request slot and separate state machine without a demonstrated need. +- Low-level `CursorWebSocketSendLoop.resolveSchema(name, timeoutMillis)` uses + the cache or issues DESCRIBE; `refreshSchema` forces discovery for subsequent + conversion revalidation. Return known or confirmed-missing immutable + responses. Existing typed schema errors distinguish access denial, + unavailable lookup and unsupported capability/size. Preserve interruption. + Neither method changes a row, binding, buffer, SF log or ACK watermark. +- The deadline includes queueing, connection wait, control send and response. + Loss releases the current waiter; a later request can wait for reconnect + within its own deadline. Installing a replacement connection clears latest + entries, not pinned snapshots or a newer unsent request. Close and terminal + exit release waiters before waiting for worker shutdown. IDs are never + reused; late or unmatched replies are discarded, not opportunistically cached. +- Cache known and confirmed missing results. Do not cache denial/unavailability; + forced refresh evicts the previous entry on admission instead of duplicating + cleanup across every failure branch. A failure never restores that entry; + independently received newer feedback remains usable. Preserve known entries + during an outage for offline row construction. Use QuestDB-compatible character folding, not + locale-sensitive whole-string lowercasing. +- A proposed SCHEMA/durable-ACK discriminator collision was disproved before + implementation: `KIND_SCHEMA=2` is inside the QWP control envelope, whereas + durable ACK starts with status 2. The full control message starts with QWP + magic. Strict envelope routing needs no protocol change or decoder fallback. +- Root and both reviewing SOL passes verified server response ordering: + `QwpIngressUpgradeProcessor.handleSchemaControl` sends its sampled response + before pending ACKs; blocked responses finish before later response handling; + `trySendAck` and `sendErrorResponse` sample current metadata when serializing. + Pending feedback retains names rather than stale snapshots. Therefore use + receive order; a client wire-sequence barrier would incorrectly suppress + newer metadata attached to an ACK for older data. No per-table revision + comparison or new server ordering field is needed. +- The existing WebSocket send path already handles whole-frame partial writes. + Send one control request at a loop boundary with its remaining deadline; + recycle after a transport/partial-send failure before sending another frame. + Do not introduce a second socket owner or send-time data transformation. + +Validation brief: public loop/network tests for cold/hit/refresh, case aliases, +missing versus failures, feedback/invalidation ordering, control/durable ACK +interleaving, timeout/cancellation/late replies, connection and close lifecycle, +and unchanged data sequence/watermarks. Real-server E2E must obtain metadata +through this loop, publish bound data, alter INT to LONG, receive updated cache +metadata through ACK, clear/rebind the same buffer, and verify exact wire +identities and SQL values. No reflection, private-field inspection, test-only +production hooks or fixed sleeps. Client/server Maven runs remain serialized +per repository, with repository-local temporary paths and installed standalone +client artifact provenance; never use the server local-client profile. + +Performance boundary: the hardware-first skill informs bounded I/O work and +allocation/ownership scrutiny. This is functional integration, not a measured +optimization. No isolated schema-coordination workload has yet established +throughput, latency or allocation cost; do not claim a speedup or zero overhead. +Keep the benchmark study separate from this bounded implementation. + +### Iteration 2.7 implementation and review checkpoint — 2026-09-10 + +The client implementation adds package-private `QwpSchemaCoordinator`, the two +documented low-level loop methods, and a byte-array WebSocket send overload +that reuses the existing whole-frame send path. No ordinary Sender setter, +row/buffer ownership or SF engine change was needed. The coordinator retains +one request object and its sending-client identity; there is no separate epoch, +atomic request state machine, callback adapter, subscription or sequence barrier. + +Root and independent SOL review corrected draft lifecycle races before final +validation: closed coordinators must reject cache hits and later feedback; +connection loss must be reported once through the common reconnect boundary; +pre-completing a failed send could otherwise cancel a subsequent request twice. +Close releases waiters before joining the I/O worker. The control-send path +propagates JVM errors to the existing fatal-exit handling. Deadline arithmetic +uses elapsed nanoseconds rather than assuming `nanoTime()` has a positive +origin. Request validation/ID assignment happens before publication, and table +name length is bounded before normalization. + +Cache storage is lazy. Invalidation drops the map reference rather than +walking up to a million entries while holding the monitor. Feedback is decoded +completely before publication, with bounded per-name monitor work; a close +guard prevents late publication. Ordinary responses without metadata bypass +the coordinator, and an idle loop checks a volatile pending flag without +taking the coordinator monitor. These are source-level ownership/work bounds, +not measured throughput, latency or allocation results. + +Independent SOL source/adversarial review accepted the final production code +and test refinements with no remaining reported blockers. Root additionally +required explicit failed-refresh eviction coverage and a genuinely negotiated +durable-ACK test, not merely injection of a stray durable ACK into ordinary mode. +The real-server test uses existing one-byte send fragmentation, real cursor +engine/loop APIs, exact INT/LONG wire identities, ACK-derived cache lookup, +same-buffer clear/rebind and SQL values `42`, `43`, `2147483648`. + +Client gates verified by root: + +- Final focused coordinator/lifecycle suite: **12 tests**, zero failures/errors/ + skips. Log: `core/target/schema-coordination-2.7/logs/focused-post-coverage.log`. +- Final canonical gate: **352 selected tests plus two packaging checks**, zero + failures/errors/skips. Log: + `core/target/schema-coordination-2.7/logs/canonical-final.log`. +- The earlier 351-test canonical gate and installation succeeded before the + final test-only refinements. The main artifact was not rewritten while the + server tests ran. Workspace and installed standalone client JAR SHA-256: + `20308e3deb22bbf2a66355afb8fcc9529022cb8b4499c4e5730ee8b2c0fe5d8f`. + +The initial client compile failed on incorrect existing getter names; the first +combined run also exposed an invalid never-started-loop test setup (null client +without a reconnect factory). A later test compile had incorrect FSN getter +names. These were corrected, their logs retained, and they are not acceptance +evidence. No reflection, private-field inspection, test-only production hook +or fixed sleep was added. Constructor/close failure cleanup in the real-server +fixture was corrected to use try-with-resources. + +The client canonical command is the iteration 2.6 selector plus +`CursorWebSocketSchemaCoordinatorTest,CursorWebSocketSchemaLifecycleTest`, with +`-Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-coordination-2.7/tmp`. +It uses the same `mvn -B -pl core ... -Dmaven.javadoc.skip=true verify` lifecycle. +Focused runs select those two new classes with `test`; installation uses +`mvn -B -pl core -DskipTests -Dmaven.javadoc.skip=true install`. +Surefire confirms the absolute project-local temporary directory. + +The full one-million-entry eviction/resource stress fixture was **not run**. +The fixed ceiling and eldest-entry removal are source-reviewed; runtime memory +and latency at that scale remain unmeasured. This does not weaken the configured +ceiling or justify adding a test-only production capacity switch. + +Server focused/aggregate/repeat acceptance remains pending at this checkpoint. + +### Iteration 2.7 final acceptance — 2026-09-10 + +Accepted by root after independent SOL source/adversarial review, final test +review and direct verification of the client/server logs and artifact hashes. +This accepts the lookup/cache coordination slice, not ordinary Sender activation +or the complete schema-directed encoding feature. + +- Client: **352 selected tests plus two packaging checks**, all green; the + **12** new coordinator/lifecycle tests are included in 352. Final evidence: + `core/target/schema-coordination-2.7/logs/canonical-final.log`. +- Server fragmented-response focused gate: **8 tests**, all green. Evidence: + `core/target/schema-coordination.2.7-server/focused-fragmented.log` in the + server repository. An earlier unfragmented focused run also passed eight + tests; the fragmented run supersedes it for acceptance. +- Server canonical gate: **281 tests**, all green. Evidence in the same server + directory: `canonical281.log`. +- Three server feedback/conversion repeats: **20 tests each**, seeds `1/2`, + `42/43`, `1234567/7654321`, all green. Evidence in the same directory: + `repeat-1-2.log`, `repeat-42-43.log`, `repeat-1234567-7654321.log`. + +All accepted runs have zero failures, errors and skips. Exact server selectors, +JVM arguments and run-specific temporary paths are recorded in the +[server run manifest](/home/jara/devel/oss/questdb/core/target/schema-coordination.2.7-server/run-manifest.md). +Root verified the final Surefire classpath uses the installed standalone +client, the final seed pair and the absolute repository-local temporary path. +The installed and workspace JAR hashes remained +`20308e3deb22bbf2a66355afb8fcc9529022cb8b4499c4e5730ee8b2c0fe5d8f` +before and after server validation. The server local-client profile was not +used. Main production bytes were unchanged by the final test-only refinements. + +Both worktrees pass `git diff --check`. HEADs remain client +`981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`. Existing unrelated changes were +preserved. No server production code changed in this iteration; only the +existing evolution E2E was extended there. No commits or pushes were made. + +Reflection: the existing server serialization guarantee removed the need for +client ordering barriers, and actual connection identity removed the need for +an epoch counter. The coordinator fits the existing I/O owner; no second +transport, generic queue or owning row facade was added. Subsequent slices +should reuse this path for remaining conversions and producer-owned +row-boundary adoption, not add another lookup/cache layer. Full million-entry +stress and performance characterization remain explicitly unmeasured, as noted +above. Do not activate a subset of ordinary Sender setters or advertise the +full extension before the conversion and inference contract is coherent. + +## D019 — Iteration 2.7.1: reject stale replies before schema parsing + +Date: 2026-09-10. Status: accepted after client/server validation and independent review. + +A fresh SOL review after D018 acceptance identified an uncovered case: the I/O +handler decoded the entire schema body before correlating its request ID. A +malformed but identifiable reply to an expired lookup could therefore recycle +the connection and fail a newer lookup. The existing late-reply test supplied +only valid metadata. The earlier green gates did not establish this case. + +The user approved the smallest correction using existing request state. Root +owns the brief, design/journal and acceptance; SOL owns implementation and +regression tests, a separate SOL pass owns independent review, and the server +validation owner reruns existing real E2Es against the newly installed client. + +Decisions: + +- Factor fixed control-envelope validation into a small protocol helper that + safely returns the positive request ID without parsing the schema body. + Reuse validation in the full decoder rather than maintaining divergent + header parsers. Header, kind and ID occupy 21 bytes; this can identify an + absent-result reply, but the full schema decoder must still reject it. + Unsafe/unidentifiable framing retains the existing connection-failure path. +- Use one synchronized, side-effect-free predicate over the existing request, + sending-client identity, ID and deadline. Check before body decoding and + again if decoding fails. Keep decoding outside the monitor so timeout and + close are not delayed by that work. Keep the existing successful-response + check before metadata publication. +- The final positive predicate check accepts a malformed reply as a live + protocol failure. Cancellation before that check wins and the stale error + is ignored; cancellation afterward does not retroactively undo the accepted + protocol failure. No claimed-request state, epoch or additional counter is + needed. +- Reject the proposed alternative of pre-completing the request before + calling `fail()`: it would wake caller A, allow request B to queue, and then + let the common connection-loss handler cancel B too. The existing timeout + and connection-loss paths retain sole ownership of their notifications. +- No new classes, queues, callbacks, protocol version or capability changes. + Ordinary Sender activation, schema conversion, cache policy and SF/ACK + semantics remain outside this fix. + +Validation starts with observable regressions against the unfixed code, with +the failing log retained. Cover malformed identifiable expired replies both +without a pending lookup and while a newer lookup waits; the connection and +new lookup must remain usable. Current malformed bodies and unidentifiable +headers must still fail. Add fixed-header/ID boundary checks while preserving +full-decoder truncation and feedback tests. Use real scripted-peer traffic, +bounded waits and observable results, not reflection, test-only production +hooks or artificial parser delays. The precise timeout-during-decoding race +also requires source review of the post-failure predicate; do not invent a +deterministic runtime claim if that scheduling point is not exercised. + +The hardware-first skill was read for this I/O-sensitive change. Its bottleneck +study is not a fit for this narrowly authorized correctness repair; no +production optimization is attempted. The functional fallback is unchanged +ownership, bounded header inspection, parsing outside the monitor and no +performance or allocation-improvement claim. No benchmark study or broader +performance work is added to this fix. + +### Iteration 2.7.1 client acceptance — 2026-09-10 + +Root and independent SOL review accepted the production fix and strengthened +regressions. Server validation remains pending at this checkpoint. + +The initial behavioral RED ran after the unused header helper was factored in, +but before changing handler/coordinator behavior. Exactly the two new +stale-malformed-reply tests failed because the old handler decoded their bodies +and triggered connection loss. The matching-active malformed case still passed. +Evidence: `core/target/schema-reply-fix-2.7.1/logs/initial-red.log`. + +Review strengthened two tests before final acceptance: + +- Sending a stale frame immediately before a new lookup did not prove the I/O + thread consumed it with no pending request. A following schema-feedback frame + on the same connection now provides an observable barrier: zero-time cache + probes cannot send a lookup, and observing the feedback with request count + still one establishes receive order. Only then does the next lookup begin. +- A schema-unavailable error alone could have come from the lookup deadline. + The matching-malformed test now uses an effectively unbounded lookup deadline + and separately waits, within a bounded test deadline, for the public transport + error. This also respects the ordering between waiter release and terminal + error publication. New expiry tests allow one second for request delivery + instead of depending on the I/O thread running within 25 ms. + +Final client gates, all with zero failures, errors and skips: + +- Focused codec/lifecycle: **20 tests**, `focused-strengthened.log`. +- Coordinator/lifecycle repeats: **15 tests each**, `repeat-1.log`, + `repeat-2.log`, `repeat-3.log`. +- Canonical: **356 selected tests plus two packaging checks**, + `canonical-strengthened-final.log`. Earlier `canonical.log` and + `canonical-final.log` predate the final test strengthening and are superseded. +- Installation: `install.log`. + +All logs are under `core/target/schema-reply-fix-2.7.1/logs/`. Runs were serialized +and used the absolute client-repository-local `core/target/schema-reply-fix-2.7.1/tmp` +directory, verified in Surefire XML. The canonical command retains D018's +selector and `verify` lifecycle. No test classes were removed from the gate. + +The workspace and installed standalone client JARs both hash to +`b5b2f5300c7d391dae99b7f9904848c13cc77d8c22c4c4d9f11f06d86de06f94`. +The installed artifact is frozen during server validation. Production changes +are limited to the protocol helper, the existing coordinator predicate and the +I/O response handler; only protocol/lifecycle tests changed alongside them. +The precise cancellation-during-decoding interleaving remains source-reviewed, +not deterministically forced by a runtime test. No new state or test hooks were +needed. Both worktrees pass `git diff --check`; no commits or pushes were made. + +### Iteration 2.7.1 final acceptance — 2026-09-10 + +Accepted after root independently verified the final client/server logs, +standalone artifact hashes and Surefire runtime configuration, alongside SOL's +independent source and adversarial test review. + +- Server focused discovery, authorization, limits, feedback and evolution: + **19 tests**, all green, `logs/focused.log`. +- Server canonical: **281 tests**, all green, `logs/canonical281.log`. +- Server feedback/conversion repeats: **20 tests each**, seeds `1/2`, `42/43`, + `1234567/7654321`, all green, `logs/repeat-1-2.log`, + `logs/repeat-42-43.log`, `logs/repeat-1234567-7654321.log`. + +These server logs live under +`/home/jara/devel/oss/questdb/core/target/schema-reply-fix.2.7.1-server/`. +Exact selectors and JVM arguments are recorded in the +[server run manifest](/home/jara/devel/oss/questdb/core/target/schema-reply-fix.2.7.1-server/run-manifest.md). +All accepted runs have zero failures, errors and skips. The final Surefire +classpath uses the installed standalone client, not the local-client profile; +its final seed pair and absolute repository-local temporary path were checked. +The workspace and installed client JARs retain the `b5b2f530...` hash recorded +above before and after server validation. + +No server source changes were needed. Both worktrees pass `git diff --check`. +HEADs remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`; unrelated changes were preserved. +No commits or pushes were made. + +Reflection: the missing step was checking existing request identity before +interpreting a reply, not adding lifecycle state. Keeping that check +side-effect-free also avoids premature waiter notification. Most review-driven +changes were to make test ordering and failure assertions conclusive. This +accepts only the stale-reply repair; ordinary Sender activation and remaining +conversion/row-boundary work remain outside this iteration. + +## D020 — Iteration 2.8: typed microsecond/nanosecond conversion + +Date: 2026-09-10. Status: accepted after independent review and client/server validation. + +The user approved proceeding with timestamp conversion. Root recommended a +bounded conversion slice, with real-server tests from the beginning. Independent +SOL client, server and adversarial reconnaissance narrowed the slice to typed +wire-unit conversion; the broader timestamp family remains unfinished. + +Frozen scope: + +- Add `QwpSchemaBinding.timestampColumn(name, long, ChronoUnit)` for MICROS and + NANOS inputs into known, non-designated TIMESTAMP_MICRO/TIMESTAMP_NANO columns. + Encode the target's timestamp wire type and converted value directly. +- Reuse the existing timestamp drivers after the two-unit support check. No new + converter framework, binding lifecycle, buffer owner, lookup state or protocol + change. Caller-owned row cancellation and column rollback remain unchanged. +- Nanos-to-micros uses Java division by 1000, truncating toward zero, not floor. + Micros-to-nanos rejects values outside the server's exact multiplication + bounds. Same-unit conversion preserves the supplied bits. +- Timestamp cursor nullness is bitmap-only. Do not copy the separate LONG-input + source-null rule: supplied `Long.MIN_VALUE` is rejected for micro-to-nano, + becomes `-9223372036854775` for nano-to-micro, and reads as SQL NULL when + stored unchanged. Explicit nulls through the existing null-input path and + omissions use the target null representation. Verify bitmap/no-bitmap cases. +- Preserve first-value-wins before input-unit validation or conversion. + Unsupported non-null units report `UNSUPPORTED_FEATURE`; a null unit and + arithmetic overflow report `INVALID_VALUE`. These errors are not flush retries. +- Reject designated timestamp targets at the shared target-support boundary, + including the null-input path. Their bounds and server-assigned-time semantics + belong with producer-owned row completion, not this ordinary-column slice. +- Defer other ChronoUnits, `Instant`, DATE, text parsing, LONG-to-timestamp, + `at`/`atNow`, automatic row-boundary adoption and ordinary Sender activation. + This is a component extension, not complete timestamp support or a release. + +Source authority is QWP ingress, not SQL casts: + +- Server `QwpWalAppender` routes timestamp precision mismatches to + `WalColumnarRowAppender.putTimestampColumnWithConversion`; matching precision + uses the existing direct-copy path. +- The conversion method checks multiplication bounds and divides toward zero. + `QwpTimestampColumnCursor.advanceRow` obtains nullness only from the bitmap. + The initial server reconnaissance mistakenly treated the primitive minimum as + source-null; root cross-check and independent review corrected that before + implementation. +- Existing `QwpWebSocketSender.toMicros` multiplies wider units unchecked; + its `Instant` overload truncates to micros. Reusing the more permissive + timestamp-driver API without a scope gate would silently tighten overflow + handling or retain extra precision. Those changes require a separate + compatibility decision. Timestamp-to-DATE is not a supported server wire pair. + +Validation brief: shared exact-value TSV vectors across client and server, +covering all four unit/target pairs, negative fractional cutoffs, multiplication +endpoints and adjacent failures, MIN/MAX, explicit/omitted nulls and unrelated +bitmap rows. Assert target wire types, exact values and SQL stored epoch values; +compare with legacy timestamp-wire ingestion rather than SQL conversion rules. +Add A/invalid-B/C rollback, removal of failed-row-only columns, cross-setter +duplicates, unsupported/name/parameter/designated failures and buffer lifecycle +tests. Exercise plain/compressed timestamps where the existing fixtures permit. +Keep existing conversion, lookup, negotiation and replay regression gates green; +extend persisted target-byte coverage without adding another transport owner. +No reflection, private-state tests, test-only production hooks or fixed sleeps. + +Root owns the design, journal, scope and acceptance. SOL owns client code/tests +and serialized client Maven; a second SOL owns server tests and serialized server +Maven; a third independently reviews source, arithmetic and observable coverage. +The client owns the shared corpus. Coordinate installation so server tests use +one frozen standalone Maven JAR, never the server's local-client profile. +Task evidence directories are client `core/target/schema-timestamps-2.8/` and +server `core/target/schema-timestamps.2.8-server/`, with absolute local JVM temps. + +Baseline client HEAD is `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD is +`12a33d651e51e2682e7a448c8db5168fc72dfad3`. Workspace/installed standalone client +JARs both hash to +`b5b2f5300c7d391dae99b7f9904848c13cc77d8c22c4c4d9f11f06d86de06f94`. +Both repositories contain prior accepted, uncommitted work. Preserve it and the +unrelated server Rust work; no commits or pushes are authorized by this slice. + +The hardware-first skill was read because this extends the write path. Its +bottleneck/optimization gate is not a fit for a functional conversion extension; +use the established functional fallback, with no production optimization or +performance/zero-allocation claim. Reuse existing buffer ownership and conversion +utilities. Hardware cost and scale measurements remain unmeasured, not implied +by passing functional tests. + +### Iteration 2.8 client acceptance and early real E2E — 2026-09-10 + +Root and independent SOL review accepted the bounded client implementation. +`timestampColumn` reuses the two existing drivers, writes the exact target type +and adds no state. Full-type matching preserves nano precision bits; unsupported +targets and duplicate suppression precede arithmetic. The shared designated +guard also rejects the existing null-input path. Caller-owned row lifecycle, +ordinary Sender, protocol, cache and replay implementation are unchanged. + +The shared corpus grew to **48 vectors**. Root and the independent reviewer each +checked it using exact BigInt arithmetic rather than production conversion +utilities: zero mismatches. Client/server resource mirrors are byte-identical, +SHA-256 `696d832cd692d5ff8765ac265070a8af6fcfcdc8c4cc87dc5967d72f4d946ab6`. + +Client gates have zero failures, errors and skips: + +- Focused binding: **26 tests**, `logs/focused-final.log`. +- Final canonical: **359 selected tests plus two packaging checks**, + `logs/canonical-final.log`, with the final 48-vector resource. +- Installation: `logs/install.log`; workspace and installed standalone JARs + hash to `909f1e039f6528067545b56477bdb8beab829a7c37f8151e9fb6ddfe6ff960ce`. + The final client verify ran without reinstalling while server validation used + the frozen installed artifact; both hashes remained equal afterward. + +Client logs are under `core/target/schema-timestamps-2.8/`. The canonical selector +is D019's unchanged selector: the three new tests live in `QwpSchemaBindingTest`. +Surefire confirms the absolute client-repository-local task temporary directory. +The first two focused runs failed because new test readers omitted timestamp +encoding/bitmap bytes and expected a never-bound column in the sparse layout. +These were fixture mistakes, not product failures or a claimed behavioral RED; +their logs remain as development evidence. The existing layout/encoder was not +changed to satisfy them. + +The first real-server timestamp E2E passed **3 tests** in +`core/target/schema-timestamps.2.8-server/logs/focused.log`. The expanded source +then passed **6 tests** in `logs/focused-final.log`, adding schema-bound MIN and +bitmap cases, actual Gorilla encoding and recovered timestamp delivery. + +Root review requested exact per-row wire and SQL checks for the compressed case, +using nanos input with a nonzero sub-micro remainder. Count/min/max alone could +hide incorrect middle values. The SF smoke proves exact target encoding before +persistence and recovered delivery with the expected value, not byte-identical +replay by itself. Existing generic replay tests separately check byte identity; +the independent review's stronger wording was corrected. Final server gates +will run after these small test refinements. No server production change is +needed, and no performance claim follows from these functional results. + +### Iteration 2.8 final acceptance — 2026-09-10 + +Accepted by root after independent SOL source/adversarial review and direct +verification of final logs, corpus equality, artifact hashes and runtime +configuration. All accepted runs have zero failures, errors and skips: + +- Client canonical: **359 selected tests plus two packaging checks**, + `core/target/schema-timestamps-2.8/logs/canonical-final.log`. +- Server refined timestamp E2E: **6 tests**, `logs/focused-final2.log`. + This supersedes the earlier three-test and six-test focused runs for final + test-source acceptance. +- Server canonical: **287 tests**, `logs/canonical287.log`. +- Server conversion/feedback repeats: **26 tests each**, seeds `1/2`, `42/43`, + `1234567/7654321`, logs `repeat-1-2.log`, `repeat-42-43.log` and + `repeat-1234567-7654321.log` under `logs/`. + +Server evidence lives under +`/home/jara/devel/oss/questdb/core/target/schema-timestamps.2.8-server/`. +Exact selectors, JVM flags and temporary directories are recorded in the +[server run manifest](/home/jara/devel/oss/questdb/core/target/schema-timestamps.2.8-server/run-manifest.md). +The final Surefire report identifies the installed standalone client, final +seed pair and absolute task-local JVM temp path. No local-client profile was +used. The workspace and installed JARs retained the full `909f1e03...` hash +recorded above throughout server validation, and the two 48-vector corpus +files remained byte-identical. + +The production change is limited to `QwpSchemaBinding`: one timestamp input +method, existing driver reuse, exact target wire mapping and the shared +designated-target support guard. The client binding tests, shared timestamp +corpus and new server E2E test provide the additional coverage. No server +production change, new converter abstraction, state or buffer owner was needed. +Both worktrees pass `git diff --check`. HEADs remain the baseline revisions +recorded above; prior/unrelated work is preserved. No commits or pushes were made. + +Reflection: typed wire-unit conversion is a useful complete subproblem. It +does not justify claiming all timestamp overloads work. Source-backed review +prevented an unchecked assumption about timestamp source nulls and prevented +existing driver reuse from silently redefining larger-unit/Instant behavior. +Wire and SQL checks establish conversion before ingestion; exact per-row +compressed checks and recovered-delivery coverage supplement that contract. +The next timestamp work needs an explicit overflow/precision decision for +broader input normalization. Designated timestamp handling remains tied to +producer-owned row completion, and ordinary Sender activation remains deferred +until the full conversion/inference contract is coherent. + +## D021 — Decide broader timestamp input semantics before implementation + +Date: 2026-09-10. Status: contract approved by the user; iteration 2.9 accepted. + +The user asked to continue after D020. Root inspected the next timestamp slice +and requested independent SOL compatibility review. The design explicitly +requires an overflow/precision decision before implementing broader units and +`Instant`; continuing implementation does not silently resolve that product +choice. No production code or tests changed in this step. + +Independent SOL review agrees with the recommendation and the need for explicit +approval. It also confirmed the intermediate-overflow issue in both timestamp +drivers and the need to preserve the existing unit allowlist. + +Current source establishes two differences from direct target conversion: + +- `QwpWebSocketSender.toMicros` multiplies MILLIS through DAYS unchecked. + For example, `Long.MAX_VALUE` seconds wraps to `-1000000` microseconds instead + of reporting overflow. Root verified the arithmetic independently with BigInt. +- Its `Instant` overload first converts to micros, even for a server nano + column. An instant 999 nanoseconds after the epoch consequently becomes zero + on that path; direct nano conversion would retain 999. + +Root recommends the following contract for schema mode only, subject to user +approval: + +- Reject inputs outside the target timestamp's range with `INVALID_VALUE` + before append, rather than preserving accidental arithmetic wraparound. +- Preserve all `Instant` precision supported by the target: retain nanos for + TIMESTAMP_NS, and retain existing `Instant`-to-micro rounding for TIMESTAMP. + Do not change the already-approved primitive NANOS-to-MICROS truncation rule. +- Support the existing QWP unit set: NANOS, MICROS, MILLIS, SECONDS, MINUTES, + HOURS and DAYS. Do not accidentally admit additional units through the + timestamp drivers' generic Duration fallback. +- Preserve duplicate suppression before conversion/argument validation. A null + `Instant` reports `INVALID_VALUE`, not an implicit timestamp null. +- Keep old-server/legacy behavior, designated timestamp handling and ordinary + Sender activation outside this slice. The changed overflow acceptance and + added nano precision must be documented as intentional schema-mode behavior. + +Implementation warning discovered before delegation: the current timestamp +drivers multiply epoch seconds with `Math.multiplyExact` before adding the +positive fractional part. Near the negative limit that intermediate product +can overflow even when the final timestamp fits. For example, epoch seconds +`-9223372037` and nano fraction `145224193` represent `Long.MIN_VALUE + 1` +nanoseconds. Root's BigInt check confirms the final value is representable; +blindly reusing `NanosTimestampDriver.from(Instant)` would reject it. If this +contract is approved, require range-correct arithmetic and boundary tests for +the final combined value, without introducing BigInteger into the write path. + +The next step is one user decision on the recommended schema-mode overflow and +precision contract. The accepted 2.8 artifact and all existing behavior remain +unchanged while that decision is pending. + +### Approval and iteration 2.9 scope + +The user approved the recommendation with "ok". Design revision 22 now records +the strict target-range and full-nanosecond `Instant` contract. The pending +decision language above records the earlier pause, not the current status. + +Implement only broader units and `Instant` on the existing non-owning binding. +Reuse the checked primitive drivers after an explicit unit gate, but keep the +range-correct `Instant` arithmetic local to schema conversion. Do not change the +shared drivers, legacy Sender, designated timestamp handling, protocol or row +ownership. This avoids broadening a schema-only decision into unrelated APIs. + +SOL assignments: client implementation/component tests and serialized client +builds; separate real-server E2E/shared vectors and serialized server builds; +independent arithmetic, compatibility and adversarial review. Root owns design, +integration and acceptance. Preserve the existing 2.8 corpus and add explicit +new input vectors, wire assertions, SQL boundary/precision checks, local-error +rollback, legacy comparisons and an `Instant` recovered-delivery case. + +The hardware-first performance skill was consulted because this changes a +write path. This is a functional correctness increment, not a bottleneck or +optimization study; its hardware-measurement gate cannot establish this feature's +contract. Keep arithmetic bounded and avoid new per-write state or BigInteger. +Hardware limits, throughput and allocation rates remain unmeasured; make no +performance improvement or zero-allocation claims. Production optimization is +outside this slice. + +### Simplification and pre-test review + +The author initially proposed checking floor-divided long bounds and handling +the lowest representable second separately. Root recommended a smaller formula: +for negative seconds, evaluate `(seconds + 1) * scale + (fraction - scale)` +with checked multiplication/addition; otherwise evaluate `seconds * scale + +fraction` with the same checks. The normalized fraction is in `[0, scale)`, so +both terms in the negative branch are non-positive. An intermediate underflow +therefore cannot hide a representable final result. No special boundary table +or new converter state is needed. + +SOL independently checked the argument and boundary arithmetic. Root also +compared checked arithmetic with an exact BigInt oracle over 200,140 boundary +and deterministic sampled cases at both scales (seed 20260910): zero mismatches. +This is an independent arithmetic check, not a substitute for component or SQL +tests. Initial independent production review found no correctness or scope issue; +test and final acceptance review remain pending. + +### Implemented behavior and validation + +The production change is confined to `QwpSchemaBinding`: extend the primitive +unit gate and add the `Instant` overload plus one small arithmetic helper. No +new schema state, buffer owner, protocol field or send-time work was introduced. +The common timestamp drivers and `QwpWebSocketSender` are byte-for-byte unchanged +from this iteration's starting source hashes. The 48-row 2.8 unit corpus is also +unchanged. + +The new shared `timestamp-inputs.tsv` has 102 cases: zero, signed values and both +range boundaries for each newly supported unit/target pair; primitive minimum +longs; `Instant` extrema, negative fractions, target precision and exact long +boundaries. Root added two precision-edge requirements during review: the extra +999 nanos within the boundary microsecond must not cause rejection. The minimum +microsecond remains a wire-present minimum long/SQL NULL; the maximum remains a +finite maximum long. Independent SOL proleptic-Gregorian/BigInt verification of +all final vectors found zero mismatches. + +Component tests assert target wire types/values, typed errors, duplicate-before- +validation ordering, partial-row and failed-column rollback, designated-target +rejection and stale-binding rejection through the new overload. The real-server +corpus test checks each accepted encoded value before sending and the resulting +SQL value; rejected inputs fail locally. The ordinary Sender test deliberately +pins the old results: maximum-long seconds wrap to -1000000000 in a nano column, +and an `Instant` 999 nanos after the epoch becomes zero. Schema conversion does +not reproduce those legacy normalization rules. + +The existing SF test now also encodes an `Instant` as exactly 123 nanos before +append, closes the cursor engine, then drains through a recovering Sender and +checks SQL. It proves pre-persistence conversion and recovered value preservation, +not exact replayed-byte identity by itself. Existing generic replay tests retain +that separate byte-identity coverage. + +Validation used serialized Maven within each repository, an installed standalone +client JAR (never `-Plocal-client`) and absolute project-local JVM temp directories. +Root verified final log results, matching corpora, stable main JAR hashes and +unchanged legacy/common-driver source hashes. All accepted runs have zero +failures, errors or skips: + +- Client focused: 89 tests with the final corpus; 29 binding tests after the + final stale-binding assertion. +- Client canonical after final review: 362 selected tests plus 2 packaging checks. +- Server focused: 8 timestamp E2E tests, including the existing six scenarios. +- Server canonical: 289 selected tests. +- Server seeded repeats: 28 tests each with seeds 1/2, 42/43 and + 1234567/7654321. +- Both repositories: `git diff --check` clean. + +Exact commands, selectors, runtime identities and log paths: + +- Client: `core/target/schema-timestamps-2.9/run-manifest.md`; + final broad log `logs/canonical-post-review-final.log`. +- Server: sibling `questdb/core/target/schema-timestamps.2.9-server/run-manifest.md`; + final broad log `logs/canonical289.log` and three `logs/repeat-*.log` files. +- Client HEAD: `981bdb02a471f3b290c89b8e78cbc422610e329e`; + server HEAD: `12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with the existing + schema worktree changes. JDK: Corretto 25.0.4; Maven: 3.9.11. +- Workspace/installed main JAR SHA-256: + `5ae6591c5ba90a29f4b743bfba0259488b4e16c0c76f59a09b29def745de08e2`. +- Both copies of the new corpus SHA-256: + `f46a23969089a41ea38bf1d801f0176e8c2fd9fef9a91c71f471a64deae169df`. + +An initial focused client run exposed an obsolete 2.8 test expectation that +MILLIS was unsupported; it now tests unsupported WEEKS. This was a fixture update +for the approved scope, not a production regression. A server run was interrupted +during compilation while the final corpus additions were synchronized; its +replacement completed successfully. Neither superseded run is counted as an +acceptance gate. The main JAR was not reinstalled after test-only additions; +server tests use its unchanged production bytes and their final mirrored corpus. + +Final independent source/test review accepted with no findings after the corpus +and new overload guards were synchronized. A second SOL adversarial review of +the client implementation also found no issue. The client author independently +cross-reviewed the server assertions and found no false-green gap against the +stated scope. That review confirmed the limited SF claim above and distinguished +the two explicit legacy-output tests from exhaustive legacy equivalence; the +broader unchanged-code claim also rests on the source-hash checks. + +Root accepts iteration 2.9. No commits or pushes were made. This remains a +conversion component, not activated schema mode in ordinary Sender. Timestamp +text parsing, designated timestamp row completion, remaining conversion families, +inference, automatic row-boundary adoption and coherent Sender activation remain +explicitly unfinished. The useful simplification is the bounded arithmetic +helper and reuse of the existing primitive drivers, not a new converter framework +or duplicated row lifecycle. + +## D022 — Validate Sender integration prerequisites before activation + +Date: 2026-09-10. Status: source-backed readiness review complete; activation deferred. + +The user explicitly required sufficient context and assumption validation before +integrating senders. Root traced the current Sender build, row completion, +rollback, buffering, encoding, publication and schema APIs against design +revision 22. Three SOL agents independently inventoried setter/server conversion +coverage, negotiation/recovery and adversarial row/schema transitions. No +production code, tests or build artifacts changed; no tests were rerun. This +journal entry records the findings under the standing decision-log agreement. + +### Confirmed gaps, not assumptions of readiness + +- **The binding is not a replacement for the Sender write surface.** Its public + inputs are LONG, STRING, UUID and timestamp overloads; non-null STRING currently + converts only to UUID. Even ordinary STRING/VARCHAR, SYMBOL, BOOLEAN and DOUBLE + identity writes are not covered. Compare `QwpSchemaBinding.java:95-243,324-373` + with `QwpWebSocketSender.java:1118-1248,1571-1738,2285-2484,2892-3017` and the + server's `QwpWalAppender.java:522-885`. Complete the agreed accepted conversion + matrix, not just method signatures or identity writes, before activation. +- **Existing setters are not interchangeable for nulls.** Decimal, array and + IPv4-text nulls are early no-ops; binary null is rejected. Preserve the explicitly + retained no-op rules. Schema-mode cross-setter duplicate suppression is an + intentional change and must run before value conversion; do not accidentally + impose all legacy mismatch/validation ordering on schema mode. +- **Inference and designated timestamps are unfinished.** The binding rejects + missing tables/columns and designated columns (`QwpSchemaBinding.java:297-332, + 407-415`). Sender currently creates its inferred layout and completes rows via + `at`/`atNow` (`QwpWebSocketSender.java:1016-1055,2924-2952,3111-3127`). Confirmed + missing metadata may use inference; unavailable metadata must never do so. + `at` needs the pinned target unit; a setter-free `atNow` still needs resolution. +- **Normal construction does not yet select schema mode.** `buildAndConnect` + requests the extension only when schema is already required + (`QwpWebSocketSender.java:3378-3379`). Existing required-capability latches protect + recovery, but are not a substitute for initial mode selection. ASYNC currently + starts with no client and marks Sender connected after starting its I/O loop + (`:4047-4064,4176`); that flag does not prove a successful handshake. The future + first effective write must wait for initial negotiation. Only a successful + old-server handshake permits legacy writes; a timeout/offline server does not. +- **A table selection is not a row boundary.** Repeating the current `table()` + returns immediately (`:2924-2929`), and another row may start without calling + `table()` again. Detect the first effective operation from actual row state, + pin once, and keep that snapshot through completion/cancellation. Clear both + designated timestamp caches when replacing a layout, not only on table switch. +- **Existing flush is publication, not just sealing.** Sender retains one current + buffer per table, encodes legacy frames (`:4335-4353`), then immediately appends + through `sealAndSwapBuffer` (`:5154-5200`). `QwpTableBuffer.reset()` preserves + column definitions and binding (`:359-367`); `clear()` destroys them and data + (`:123-136`). Neither alone retains old completed rows while adopting new types. + Preserve old encoded blocks and existing pending accounting, symbol dictionary + ordering, split-frame behavior and per-table transaction/commit boundaries. +- **Refresh is available but not integrated with row errors.** The loop exposes + resolve/refresh (`CursorWebSocketSendLoop.java:1226-1239`); Sender calls neither. + `SCHEMA_CHANGED` currently appears only in the reason enum. Existing evolution + E2Es explicitly reset/clear/rebind their component buffers + (`QwpSchemaFeedbackEvolutionE2ETest.java:76-134`); they do not prove automatic + Sender adoption or whole-row retry after a cached conversion rejection. + +API presence also does not prove successful server ingestion: current client +`longArray` overloads emit TYPE_LONG_ARRAY, but this server's mapping explicitly +rejects that wire type (`QwpWalAppender.java:124-125`), with a real-server rejection +test (`QwpSenderE2ETest.java:3628-3636`). Do not count it among working baseline +writes, or infer a new LONG_ARRAY conversion contract from the method name. +Validate the design's array-input contract in its own conversion slice. + +### Shortcuts challenged during review + +Root rejected two initial SOL recommendations: routing only the implemented +setters while retaining raw/server-side conversion for the rest, and permitting +ASYNC rows before initial negotiation. Both contradict revision 22. The agents +re-read the contract and corrected their recommendations. Existing legacy rows +may survive an old-to-new upgrade only after legacy mode was established by a +successful old-server connection; they are not permission for offline guessing. + +The adversarial review also corrected an overly broad rejection of sealing on +schema change. Internal sealing is allowed; calling the current publishing +flush path unconditionally is not proven safe. A retained local encoding or +deliberately deferred publication could reuse existing machinery, but each must +prove ordering, accounting, reset/close, crash/recovery and backpressure behavior. +Do not choose a new per-table generation container or queue framework merely +because multiple encoding snapshots must coexist. Likewise, reuse the existing +request flag and mandatory-capability latches before adding negotiation state; +requesting support and requiring it are distinct semantics, not justification +for another owner or user-visible switch. + +### Observable integration acceptance sequences + +1. Buffer an INT/v10 row, learn LONG/v11, and write the next row without another + `table()` call. Verify separate old/new wire types and identities plus SQL. + Repeat with feedback arriving mid-row: that whole row stays on its old schema. +2. Exercise `at`, setter-free `atNow`, no-op/null inputs, duplicate inputs and + cached timestamp references across row/layout transitions. +3. Reject against a cached schema and refresh once: unchanged target gives the + invalid-value result; changed target/incarnation cancels the whole row and + gives `SCHEMA_CHANGED`; refresh failure gives its actual reason. Earlier + completed rows survive and the next whole-row attempt can succeed. +4. With all auto-flush triggers disabled, buffer rows across two tables and a + schema change. Prove no unintended publication/commit, correct explicit-flush + results and unchanged reset/crash behavior. Repeat with transactional + auto-flush, capacity pressure, dictionary splitting and reconnect. Preserve + existing per-table guarantees; do not claim new cross-table atomicity or + stronger delivery guarantees after partial publish failures. +5. Test successful old/new handshakes, ASYNC first-write timeout, old-to-new + upgrade with pending legacy rows, reconnect during a partial row and extended + SF/background recovery. Never guess legacy, downgrade or relabel bytes. +6. Cover ordinary working setter/target pairs and confirmed missing-table/column + inference alongside unavailable/denied metadata. No raw-value fallback or + per-setter opt-out on supporting servers. + +Recommendation: continue bounded converter/inference and row-lifecycle work +without activating normal Sender. Lock down existing Sender behavior with +observable characterization tests where needed. Public activation must land with +the complete integration matrix above, not ahead of it. Reuse the existing row +owner, schema coordinator, encoder and recovery path; choose any additional +storage/state only after those sequences establish why it is necessary. + +## D023 — Text and symbol inputs reuse target-native text encoding + +Date: 2026-09-10. Status: iteration 2.10 accepted after review and validation. + +The user asked to continue after the integration-readiness review. The next +bounded slice covers six pairs on the existing non-owning binding: STRING and +SYMBOL inputs into exact STRING, VARCHAR and SYMBOL schema targets. Leave normal +Sender, inference, designated timestamps, other conversions and activation alone. +This fills ordinary text coverage without loosening the D022 activation gate. + +Root and independent SOL source review verified: + +- The server accepts all six pairs (`QwpWalAppender.java:698-805`). STRING and + VARCHAR targets share TYPE_VARCHAR on QWP; SYMBOL retains TYPE_SYMBOL. +- Existing `ColumnBuffer.addString` encodes text into its offsets/data storage; + `addSymbol` supports both the owner-backed global dictionary and a senderless + local dictionary (`QwpTableBuffer.java:1197-1228,1855-1863`). Its constructor + documentation overstates the need for a Sender, but the implementation already + provides the needed local path. No new dictionary or owner is necessary. +- Both active client serialization paths replace individual unpaired UTF-16 + surrogates with literal `?`, preserve following characters and encode valid + pairs normally (`OffHeapAppendMemory.java:166-202`, + `NativeBufferWriter.putString`/`Utf8s.strCpyUtf8`). Preserve this behavior rather + than substituting a parser/encoder or changing shared helpers. +- Null is distinct from empty. Text value case and whitespace must not be + normalized; column-name case folding is unrelated. Distinct raw strings can + occupy separate symbol IDs and still encode to the same replacement text. +- Text writes and dictionary insertion snapshot mutable inputs using existing + encoding/copy operations. Tests must mutate a supplied CharSequence after the + setter and verify the buffered value, not inspect ownership fields. + +Implementation: extend `stringColumn` and the existing exact target-to-wire map, +then add `symbol(name, value)` with the same duplicate-first buffer lookup. +Preserve STRING-to-UUID and existing broader STRING-null behavior. The new SYMBOL +input accepts null only for its implemented text targets; reject other pairs +unless the value is already suppressed as a duplicate. Keep unknown full target +types, extension parameters and designated columns behind existing guards. + +SOL owns client implementation/component tests/shared corpus, a separate real +server E2E implementation and serialized server builds, and independent source/ +corpus/adversarial review. Root owns scope, final cross-review and acceptance. +The corpus records input UTF-16 units, expected target wire type/UTF-8 bytes and +expected SQL text units explicitly. Cover all six pairs, null/empty, Unicode, +malformed surrogate sequences, controls, dictionary collisions, duplicates, +rollback, mutable inputs and reset/clear. Also test existing global dictionary +encoding and SF recovery through production APIs, with no reflection, TestOnly +accessors, new hooks or fixed sleeps. Assert encoded bytes and real SQL, not just +successful sending. Existing corpora remain unchanged. + +The hardware-first performance skill was consulted for the write-path change. +This is functional coverage, not production optimization: reuse existing storage +and serialization and add no new state/framework. No hardware bottleneck, +throughput improvement or allocation-rate claim is made. As in D021, measurement +gates are not a substitute for conversion correctness. Use absolute project-local +JVM temp storage; `/tmp` remains nearly full. Preserve all existing work and do +not commit or push. + +Review clarified an existing encoder precondition: a Sender-owned symbol buffer +contains global IDs, so it must use the global dictionary message path. The +senderless convenience encoder uses a local dictionary and cannot encode that +buffer correctly. Do not conceal this by introducing another dictionary owner or +an automatic format conversion. The owner-backed test constructs a normal Sender, +lets the binding intern new values, then uses a matching explicit test dictionary +to assert the global delta and actual row IDs through the production encoder. +Only a prefix is registered before binding writes; registering every tested value +in advance would miss the insertion behavior. Distinct raw keys with identical +replacement text must retain distinct client IDs. + +Coverage remains deliberately separated: the client fixture proves owner-backed +global encoding; real-server component and SF tests prove the existing local +dictionary path; ordinary Sender tests establish legacy value parity. Together +these do not prove activated schema-aware Sender delivery or global SF replay. +Those remain obligations of the D022 integration gate. + +### Review and test corrections + +Independent SOL review checked all 90 vectors with a separate UTF-16/UTF-8 +oracle: zero mismatches. Root strengthened the tests to assert reset output on +the wire, enforce 90 corpus records and 15 records per conversion pair, and +preserve a non-null completed A row across failed B and completed C. Server +checks also distinguish explicit null and omission from empty text using raw +values and SQL `is null`, not the text printer's blank-cell rendering. + +The broader server suite exposed an obsolete assertion in +`QwpSchemaRowBufferE2ETest`: STRING-to-STRING had deliberately been unsupported +in the earlier UUID-only slice. Preserve that test's unsupported-column rollback +coverage using LONG-to-STRING, which is still outside this binding's supported +conversions. Do not remove the assertion or turn off the regression suite. + +The directly related `QwpTableBuffer` constructor Javadoc was corrected to +describe its global/local dictionary choice. This is the sole change to that +existing buffer class in this iteration: documentation only, with unchanged line +count, made after the client gates/install. The executable artifact remains +frozen for server validation; no reinstall occurs during server tests. + +Test-authoring failures remain in the run manifests: Java-11-incompatible switch +expressions in a new helper, an incorrect close-flush configuration key, and the +SQL null-text printing expectation. These did not require production fixes. +The shared TSV intentionally has trailing delimiters on its six empty-text +records: these retain seven fields and must not be stripped as whitespace. + +### Final validation and reflection + +All final gates passed with zero failures, errors or skips: + +- Client focused: 35 tests (29 existing binding tests and 6 text tests). +- Client canonical: 368 selected tests plus 2 packaging checks. +- Server focused: 8 tests across text and the existing row-buffer class. +- Server canonical: 293 selected tests. +- Server seeded repeats: 32 tests each with seeds 1/2, 42/43 and + 1234567/7654321. + +The canonical selectors extend D021 with `QwpSchemaBindingTextTest` on the client +and `QwpSchemaTextE2ETest` on the server; the latter also joins all three seeded +repeats. Exact commands, runtime identities and superseded-run explanations are +in `core/target/schema-text-2.10/run-manifest.md` and sibling +`questdb/core/target/schema-text.2.10-server/run-manifest.md`. Final logs are +client `logs/canonical-final.log`; server `logs/focused-final3.log`, +`logs/canonical293-final.log` and the three `logs/repeat-*.log` files. + +Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD remains +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with the existing dirty schema +worktrees preserved. Runtime: Corretto 25.0.4 and Maven 3.9.11. Builds were +serialized within each repository; server runs used the frozen installed client +JAR, never `-Plocal-client`, and absolute project-local JVM temp directories. + +- Workspace/installed main JAR SHA-256: + `572065d10739860e6b63d80f55f6834429f2b9134c360519d7e9aed094d08443`. +- Both 90-record text corpus copies SHA-256: + `b5bdb1102a6d0887c8c81acc0eabf7519b56ac809f05bc505b44b92ccdbcaba7`. +- Root rechecked unchanged Sender, column writer, global dictionary and both + UTF-8 serialization source hashes, and unchanged timestamp corpora. Restoring + only the old constructor comment in a read-only stream reproduces the exact + pre-iteration `QwpTableBuffer` source hash; all executable buffer code is + unchanged. Both repositories pass `git diff --check`; untracked Java checks + are clean, with the intentional TSV delimiters noted above. + +Independent SOL source/test review accepted the final code, corpus, dictionary +fixture and corrected old test. The client author also cross-reviewed real-server +assertions; the server author independently reviewed the client conversion and +dictionary paths adversarially, finding no defect. Root checked the actual +wire/SQL assertions, logs, corpus hashes, final server classpath/temp/seed +properties and scope boundaries rather than treating review votes as acceptance +evidence. + +Root accepts iteration 2.10. The useful simplification is reusing existing text +storage, symbol dictionaries and encoding paths: six conversions need no new +production state, generic converter framework or row owner. This is a functional +result, not a measured performance claim. Normal Sender remains unmodified by +this slice, and D022's conversion, inference, negotiation and row-boundary +integration requirements remain open. No commits or pushes were made. + +## D024 — Boolean conversion uses target values and strict ASCII parsing + +Date: 2026-09-10. Status: iteration 2.11 accepted after review and validation. + +Continue with the BOOLEAN component family, not ordinary Sender integration. +Root and separate SOL source reviews confirmed a small complete scope: +`boolColumn(boolean)` into BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING +and VARCHAR, plus `stringColumn` into BOOLEAN. BOOLEAN-to-SYMBOL and other +unimplemented pairs remain unsupported. Preserve every existing conversion and +the broader `stringColumn(null)` rule; adding a BOOLEAN wire mapping must not +implicitly authorize other input methods. + +### Source-backed contract and implementation boundary + +- `QwpWalAppender.java:555-563,608-616,698-805` and + `WalColumnarRowAppender.java:228-330` establish the nine boolean targets. + Convert directly before append: boolean, exact numeric zero/one or lowercase + ASCII text. Do not retain BOOLEAN wire for a numeric/text target and leave + conversion to the server. +- `WalColumnarRowAppender.java:1241-1268` accepts only single-character `0`/`1` + and ASCII-case-insensitive `true`/`false`. Whitespace, signs, alternate words, + embedded NUL, Unicode lookalikes and malformed surrogate sequences are invalid. + The existing client `Chars.equalsLowerCaseAscii` uses strict ASCII folding and + can be reused without allocating strings or encoding temporary UTF-8. Java + `Boolean.parseBoolean` and Unicode case-insensitive comparison have different + acceptance rules and must not replace this contract. +- BOOLEAN storage has no distinct SQL null: the server maps wire null to false. + BYTE/SHORT similarly store zero. Nullable numeric and text targets retain their + own nulls. This follows the already approved target-native missing-value rule, + not a new policy choice. Legacy boolean layouts instead materialize omitted + values as false before server conversion, giving numeric zero or text `false`. + Test both behaviors explicitly without changing legacy Sender. +- `ColumnBuffer.addBoolean` already stores ordinary boolean values; the column + writer emits a row-null bitmap and then packs only non-null value bits. + `QwpBooleanColumnCursor` advances its value index only for non-null rows. + Reuse this representation and test across both byte and 64-row boundaries. + +Only `QwpSchemaBinding` needs executable client changes: the bool setter, the +BOOLEAN target-to-wire case and the string parser case. Reuse the existing +buffer ownership, row cancellation, stale-binding guard and duplicate-first +lookup. No production changes to Sender, buffer, encoder, shared serializers, +server ingestion, lookup/cache coordination or protocol. No new conversion +framework, dictionary, fallback or feature switch. + +SOL responsibilities: client implementation plus shared corpora and observable +wire tests; separate real-server E2E implementation/builds; independent and +adversarial source/corpus/test review. Root owns scope, assumption validation, +design/journal and acceptance. Planned corpora exhaust the ASCII case variants +of both accepted words and cover invalid tokens, all nine supplied-boolean +target pairs and explicit missing-value scenarios. Tests must include local +invalid-value errors, duplicate suppression before parsing, mutable input, +reset/clear, A-invalid-B-C row rollback and failed-row-only columns, legacy +accept/reject parity, intentional omission differences and recovered target +values. Use real wire and SQL observations, no reflection or TestOnly hooks. + +The performance skill is applied as in D021/D023: this is functional coverage, +not production optimization, and no hardware bottleneck or speed/allocation-rate +claim is made. Source-level reuse is not a measured performance result. Keep +Maven serialized within each repository, install/freeze the client only after +client gates, then run server gates against that installed JAR without +`-Plocal-client`. Use absolute project-local temp paths; `/tmp` has about 621 MB +free. Preserve dirty worktrees, existing corpora and unrelated server files. +No commits or pushes. + +Initial corpus review caught two ambiguous expectations before acceptance: +numeric target results must say zero/one, not reuse boolean `false`/`true` +labels; and explicit STRING null to BOOLEAN must identify a wire null, not a +stored false bit, even though SQL reads false. Keep the wire and SQL oracles +separate. Root also required the Unicode long-s spelling of `false` as a +rejection case: it specifically detects accidental Unicode case folding, unlike +an arbitrary non-ASCII typo. Packed tests must cross 64 non-null values as well +as 64 total rows; these are different indices when nulls are interspersed. + +The final shared corpora contain 18 primitive-boolean target cases and 80 text +cases: all 50 accepted non-null ASCII spellings, one explicit null and 29 invalid +tokens. Root's independently constructed accepted-token set and target-value +oracle found zero mismatches; independent SOL review agrees. Client wire tests +use 137 rows with 69 supplied booleans, then reset to 70 all-null rows, checking +the raw null bitmap and compacted value bitmap separately. The server pattern +uses 137 rows with 93 supplied values; acceptance requires checking stored values +by row, not only aggregate counts. Float/double zero checks must compare exact +bits so negative zero cannot pass a delta-based assertion for positive zero. + +Final source review keeps the omission comparison deliberately representative: +INT and STRING explicitly contrast schema target nulls with legacy zero/text +`false`. All nine targets still have supplied-value parity tests; the packed +BOOLEAN case and existing numeric-null tests cover their separate missing-value +rules. Do not describe the representative comparison as nine-target omission +coverage. A transient expanded draft was removed before the accepted runs. + +The first client run exposed a raw null-bitmap fixture expectation error; the +first server build exposed missing required WebSocket close callbacks in the +test helpers. Fixes stay in tests, with unexpected close now failing explicitly. +The accepted server invalid-token tests send actual legacy VARCHAR frames and +require both SCHEMA_MISMATCH and the boolean-parse diagnostic. Generic failure +or timeout is not evidence of parser parity. Superseded development logs remain +identified in the run manifests. + +### Final validation and reflection + +All accepted runs passed with zero failures, errors or skips: + +- Client focused: 41 tests, including all three binding test classes. +- Client canonical: 374 selected tests plus 2 packaging checks. +- Server focused: 6 new BOOLEAN E2E tests. +- Server canonical: 299 selected tests. +- Server seeded repeats: 38 tests each with seeds 1/2, 42/43 and + 1234567/7654321. + +The canonical selectors extend D023 with `QwpSchemaBindingBooleanTest` on the +client and `QwpSchemaBooleanE2ETest` on the server; the latter also joins the +three seeded repeats. Commands, selectors and logs are recorded in client +`core/target/schema-boolean-2.11/run-manifest.md` and sibling +`questdb/core/target/schema-boolean.2.11-server/run-manifest.md`. Accepted logs +are client `logs/focused-final.log` and `logs/canonical-final.log`; server +`logs/focused-final.log`, `logs/canonical299.log` and the three +`logs/repeat-*.log` files. The initial server compile-only failure is not counted +as a test run or acceptance evidence. + +Source HEADs remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with the dirty schema worktrees +preserved. Runtime is Corretto 25.0.4 with Maven 3.9.11. Server tests use the +standalone client installed after its final gates, never `-Plocal-client`, and +absolute repository-local JVM temp directories. Root verified the final Surefire +classpath, temp path and seed properties, not only the reported counts. + +- Workspace/installed main JAR SHA-256, unchanged during server testing: + `b1aa01621ae22cfc39b15a9b1385c82834f5999ab667fb2c2f6798bdeae4a86f`. +- Both copies of `boolean-to-target.tsv` SHA-256: + `37038ba5b25a254b7b0e005ba449d3d652a5541734a98fdb9f5dc0dae0540581`. +- Both copies of `string-to-boolean.tsv` SHA-256: + `1b8952d51862f68e80c23ea06111b3e4cfdee8a87cf37c3374d57d430b288c44`. + +Root checked that the tracked production diffs in both repositories are +unchanged from the start of this iteration, along with the untracked server +schema-control source. The only executable production changes are in the +already-untracked client binding. Sender, buffer, column writer and all existing +corpora retain their pre-iteration hashes. Both repository diff checks and +explicit checks of the new untracked tests/resources are clean. + +Independent SOL review accepted the conversion code, corpora and final test +contracts; the client author also cross-reviewed the server test, and the server +author independently audited the client implementation adversarially with no +blocker. Runtime gates remain necessary: source review alone did not establish +that test fixtures compiled or that their wire expectations were correct. The +accepted tests now assert actual encoded values, typed local failures, specific +server parsing failures and exact stored results, with no private hooks or +reflection. + +Root accepts iteration 2.11. Reusing strict ASCII comparison and existing target +writers kept production changes limited to a setter, parser branch and type +mapping. No new state or conversion framework was needed, and no performance +improvement is claimed. The SF test proves conversion before persistence and +preservation of recovered values, not byte-identical replay by itself. Ordinary +Sender integration and the remaining D022 conversion/inference/row-boundary +requirements are still unfinished. No commits or pushes were made. + +## D025 — Floating-point source semantics need an explicit compatibility decision + +Date: 2026-09-10. Status: findings verified against source and real-server tests; +characterization accepted. The compatibility decision was subsequently approved +by the user in D026. This entry records the investigation, not implementation. + +The next candidate slice is FLOAT/DOUBLE inputs into BYTE, SHORT, INT, LONG, +FLOAT and DOUBLE. Before assigning implementation, root and three SOL agents +traced ordinary Sender, the column encoder, QWP dispatch and server conversion. +The twelve server-supported numeric pairs are small enough to implement directly +in the binding, but supplied-value parity has two important edge cases. + +### Source evidence and recommendation + +- Ordinary `QwpWebSocketSender.floatColumn`/`doubleColumn` use bitmap-capable + columns and append supplied values without NaN normalization. `ColumnBuffer` + and the column writer preserve the raw value bytes. Crucially, + `QwpColumnWriter.writeNullHeader` emits no bitmap when there are no omitted + values; bitmap capability is not the same as a bitmap on the wire. +- `QwpFixedWidthColumnCursor.advanceRow` checks NaN as a null sentinel only + without a bitmap. With a bitmap, a supplied NaN is non-null and reaches + `WalColumnarRowAppender.checkDoubleToInteger`, which rejects it. Without a + bitmap, BYTE/SHORT instead receive zero and INT/LONG receive SQL NULL. Adding + an omitted row therefore changes whether another row is accepted. D013 + approved a stable source-null rule for LONG minimum only, not for NaN. +- `QwpWalAppender.isFixedTypeCoercionAllowed` confirms the six numeric targets; + DATE and TIMESTAMP are not part of this floating-point slice. For integer + targets, the server casts to long and compares the double round trip, then + checks the narrower integer bounds. Exactly positive `2^63` casts to + `Long.MAX_VALUE`, whose double representation rounds back to `2^63`, so the + equality test accepts it. This is silent clamping of an out-of-range value, + not exact conversion. Fractions and infinities fail the round-trip check. +- FLOAT/DOUBLE targets use normal widening/narrowing; finite DOUBLE overflow + to FLOAT infinity and signed zero are separate behaviors. Do not accidentally + ban them with a blanket finite-value or mathematical-range rule. + +Root recommends NaN-as-source-null for supported numeric targets, consistent +with D013, and mathematical range checking for floating-point inputs to integer +targets. Preserve duplicate-first and supported-pair checks before null handling. +This means zero for BYTE/SHORT and NULL for nullable numeric targets; positive +`2^63` to LONG must fail locally instead of being clamped. Both are proposed +schema-mode exceptions to parity, not changes to legacy Sender or server +ingestion. No per-row state, batch scan, send-time conversion or framework is +needed. The exact negative LONG endpoint remains representable and follows the +target's existing SQL-null sentinel semantics; it is not a new source-null rule. + +One independent reviewer initially called the twelve-pair slice coherent under +existing parity, then withdrew that conclusion after root challenged the +bitmap-free NaN case. A second review initially conflated bitmap-capable storage +with a serialized bitmap and corrected that claim against `writeNullHeader`. +These corrections matter: choosing either always-reject or always-null for NaN +changes one working legacy context. Reviews are evidence to check, not votes. +The independent review favors preserving the clamped endpoint for compatibility; +root favors rejecting it because early validation should not silently change +an out-of-range integer. User approval must settle the deliberate exception. + +SOL server validation owns test-only real-server characterization using ordinary +Sender and exact stored values or server error messages. Root verified the +final tests and logs before presenting the decision. No client build/install, +server production change, ordinary Sender integration or new converter is part +of this investigation. The performance skill remains a functional-work guard: +no optimization, hardware bottleneck or speed/allocation-rate claim is made. +Existing worktrees and the frozen 2.11 client artifact are preserved. + +### Characterization construction and review + +Two tests were added to the existing server `QwpSenderE2ETest`, exercising +ordinary public Sender calls. NaN covers both FLOAT/DOUBLE sources and all four +integer targets, with and without an omitted row. The endpoint test checks +exactly positive `2^63`, the adjacent smaller representable source value and +the adjacent larger value, separately for FLOAT and DOUBLE. Assertions use +exact SQL values, missing-value predicates, specific SCHEMA_MISMATCH diagnostics +and zero committed rows from rejected blocks. + +The first endpoint fixture mixed FLOAT and DOUBLE setters in one legacy layout +and failed locally, before the server conversion. Root caught the invalid +assumption; separate table layouts corrected the test. Root and adversarial +review also required explicit batching settings for NaN plus omission, so a +timer cannot flush the NaN row before the omitted row is appended. A dedicated +test helper pins the row threshold above two, disables the byte trigger and +sets the interval to the largest supported value. It does not alter the shared +rejection helper or production code. The first compile after this change used +a private superclass timeout constant; an explicit test timeout corrected it. +Error-message target names use locale-independent case conversion. Failed and +superseded intermediate runs are retained and are not final acceptance evidence. + +### Final verification and handoff + +The final two characterization tests passed, with zero failures, errors or +skips. Three repeats also passed both tests, using seeds 1/2, 42/43 and +1234567/7654321. Accepted server logs are +`core/target/schema-floating-feasibility/logs/final.log` and the three matching +`logs/repeat-*.log` files. The adjacent `run-manifest.md` records the selector, +JVM arguments, temporary paths and superseded runs. An intermediate compile +was interrupted for the locale fix; it is explicitly excluded from acceptance. + +Root checked the final source, all four successful logs, the last Surefire XML +classpath/temp/seed properties and unchanged workspace/installed client JAR. +The final server source received independent adversarial SOL acceptance. No +reflection, TestOnly method or private-state hook is used. These checks establish +the two legacy behaviors; they do not validate an unimplemented schema converter +or replace its future full regression matrix. + +- Client JAR SHA-256 remains + `b1aa01621ae22cfc39b15a9b1385c82834f5999ab667fb2c2f6798bdeae4a86f`. +- Final `QwpSenderE2ETest.java` SHA-256: + `9bcb321488b05e724eb90c8e5eb3f4ea205bec3b5716f7380fbe6fe8668a5ba6`. +- Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD + remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`. Tracked production diff + hashes in both worktrees match the pre-investigation values, and the client + binding is unchanged. Both repository diff checks are clean; earlier dirty + work and the unrelated server Rust checkout are preserved. + +Root accepts the characterization, not iteration 2.12. The design explicitly +keeps the two proposed parity exceptions pending. No conversion code, Sender +activation, legacy behavior, protocol, persisted bytes or server ingestion +was changed. No client reinstall, full canonical rerun, commit or push was +performed. Next action requires the user's decision on consistent NaN missing +values and rejection of floating-point inputs outside an integer target's range. + +## D026 — Approved FLOAT/DOUBLE numeric conversion rules + +Date: 2026-09-10. Status: user approved; iteration 2.12 accepted after review and +client/server validation. + +The user answered "yes" to applying both recommended rules in schema mode while +leaving legacy behavior unchanged. NaN is source missing for supported numeric +targets, and floating-point input outside an integer target's mathematical range +fails locally. Design revision 26 makes both exceptions explicit; D025's +characterization remains the preserved legacy baseline. + +Scope: add `floatColumn` and `doubleColumn` to the non-owning binding for BYTE, +SHORT, INT, LONG, FLOAT and DOUBLE. No text formatting/parsing, decimals, arrays, +DATE/TIMESTAMP conversion, inference, row-boundary adoption or Sender activation. +Executable production changes stay in `QwpSchemaBinding`; existing buffer, +encoder, ownership and rollback remain. One shared floating numeric helper is +acceptable: widening a FLOAT input to DOUBLE is exact for all non-NaN values, +and NaNs are normalized before append. Do not duplicate twelve target branches +or introduce a converter framework merely to keep separate setter names. + +Preserve duplicate-first behavior, exact supported-pair/parameter/designated +checks before NaN normalization, and the input type in error context. NaN uses +the target missing representation, not a raw NaN payload. Integer conversion +rejects fractions and infinities and explicitly excludes positive `2^63` before +any long cast could silently saturate. Inclusive BYTE/SHORT/INT ranges and the +negative LONG endpoint remain. A finite conversion landing on INT/LONG minimum +is still a present target sentinel that reads SQL NULL, not a new source-null +policy. Floating targets retain server-equivalent non-NaN bits and rounding, +including negative zero and overflow/underflow when narrowing DOUBLE to FLOAT. + +SOL responsibilities: client binding, shared raw-bit corpus and observable +client tests; separate real-server E2E author/build owner; independent source, +corpus and adversarial test reviewer. Root owns scope, independent boundary +oracle, design/journal and acceptance. The client corpus owner publishes its +format early so server tests can be written independently. Tests must assert +wire type, actual bitmap/value bytes and stored values separately; a cursor's +sentinel interpretation alone does not prove an explicit wire null. + +Cover all twelve pairs; +/-zero, subnormal and finite extrema; integer edges +and adjacent representable floats; fractions and infinities; multiple NaN +payloads/signs; nulls with and without other omitted rows; duplicate invalid or +unsupported inputs; reset/clear/stale bindings; whole-row rollback preserving +A/C and removing failed-row-only columns; and converted bytes persisted and +recovered through SF. Legacy tests must separate preserved parity from the two +approved differences. No reflection, private hooks, TestOnly APIs or assertions +whose expected result is computed by the converter being tested. + +Run focused and canonical client gates before installing/freezing its standalone +artifact. Only then run server gates against that installed JAR, never +`-Plocal-client`. Extend the 2.11 canonical selectors with the new floating +tests and retain both D025 legacy characterization methods. Repeat the server +schema matrix with the existing three seed pairs. Keep Maven serialized within +each repository and use absolute project-local JVM temp directories. Preserve +dirty worktrees, all prior corpora and unrelated files; no commits or pushes. + +As in D024, the performance skill is used for functional-work discipline rather +than a production optimization: existing storage/writers are reused, measured +hardware behavior is unknown, and no bottleneck, throughput or allocation-rate +claim is made. Functional test counts are not performance evidence. + +### Implementation and independent checks + +The implementation uses two public setter methods and one shared floating +numeric path. It reuses the numeric target predicate with LONG conversion; +following root review, SOL consolidated the two identical allowlists. No buffer, +encoder, wire mapping, owner or server ingestion change was needed. Integer +checks use finite/whole-value validation and exact mathematical bounds, with +the LONG upper bound expressed as exclusive positive `2^63`. + +The shared corpus is frozen at 350 raw-bit cases, SHA-256 +`7194ec96006341cf5a5d9081fd75d8f1ab7a1a5813067ba80971d01df8957216`. +Its seven fields separate input bits, target type, wire type, wire bits and +stored value. Explicit wire null is distinct from a present INT/LONG sentinel +and from BYTE/SHORT's stored zero. Stored floating values use raw-bit expectations +to preserve negative zero and avoid relying on SQL text formatting. + +Root's exact-IEEE BigInt oracle found eight incorrect draft SQL expectations: +NaN-to-BYTE/SHORT encodes an explicit null but stores zero, not SQL NULL. These +were corrected before acceptance. Additional vectors cover signaling NaN, +negative fractions, LONG/INT boundaries and adjacent representable inputs, +round-to-even ties, zero/subnormal and subnormal/normal transitions, and the +finite-to-infinity narrowing threshold. Input raw bits, not approximate numeric +case labels, are authoritative: FLOAT cannot distinguish integer values already +rounded to the same input before the setter call. + +Both root's independent BigInt oracle and a separate SOL arithmetic check agree +on all 350 cases: 106 invalid, 36 explicit missing and 208 present values across +all twelve input/target pairs. Root's diagnostic script is retained at +`core/target/schema-floating-oracle-2.12/check-corpus.cjs`; its rounding logic +was separately cross-checked on 10,000 deterministic IEEE samples. That sample +validates the diagnostic oracle, not 10,000 executions of the new converter. + +Root and SOL review strengthened duplicate tests to assert retained encoded +values rather than only successful encoding. The new client tests also check +actual null bitmap/value bytes, whole-row and failed-column rollback, reset and +stale binding behavior, unsupported targets before NaN handling, and input/target +error context. Independent client review found no remaining issue. The first +focused run passed 47 tests, including all six new floating tests; broader +client and real-server validation remain the acceptance gates below. + +Server review required three additional observations before runtime acceptance: +test all 104 legacy-invalid raw frames, excluding only the two approved positive +`2^63` clamps; cover both NaN row contexts for both input types and all six +targets; and prove that a bitmap actually exists and contains zero compact +values for an all-missing column. A cursor null flag alone was insufficient, +because its bitmap-free sentinel interpretation is the original compatibility +problem. Once bitmap presence and compact value counts are checked, the existing +cursor's row flags and value reads can be reused without another wire parser. + +Stored-value checks were corrected to recognize NaN as FLOAT/DOUBLE SQL null; +BYTE/SHORT omission cases assert actual zeros, not just non-null counts. The SF +test uses literal expected integer and floating bits before persistence and +after recovery, rather than describing converted values as byte-identity proof. +Independent server source review accepted these final contracts, with runtime +verification still required. + +The first server focused run exposed an incorrect sparse-layout column index +in the new NaN fixture: only `value` was written, so its wire index is zero even +though it follows `case_id` in the SQL schema. Root caught the assumption during +review; the fixture now asserts the single encoded column's name and type. The +next focused run exposed an expected SQL-header mismatch for two unaliased +counts. These are test-fixture failures, not converter defects. Their logs are +retained separately from the final accepted runs. Source review did not replace +compilation and execution. + +### Final validation and acceptance + +All final gates passed with zero failures, errors or skips: + +- Client focused: 47 tests, including 6 new floating tests. +- Client canonical: 380 tests plus 2 packaging checks. +- Server focused: 5 new floating E2E tests. +- Server canonical: 306 tests. +- Server seeded repeats: 45 tests each at seeds 1/2, 42/43 and + 1234567/7654321. + +The client selector extends D024's 374 with `QwpSchemaBindingFloatingTest`. +The server selector extends D024's 299 with the five-test +`QwpSchemaFloatingNumericE2ETest` and both D025 legacy characterization methods +in `QwpSenderE2ETest`. The repeat selector likewise adds those seven tests to +D024's 38. Explicit method filters retain both older UUID checks in the +canonical run; adding methods to the source alone would not select them. + +Client commands and logs are recorded in +`core/target/schema-floating-2.12/run-manifest.md`; accepted logs are +`logs/focused-initial.log`, `logs/canonical-final.log` and `logs/install.log`. +Server commands and logs are recorded under sibling +`questdb/core/target/schema-floating.2.12-server`; accepted logs are +`logs/focused-final2.log`, `logs/canonical306.log` and the three +`logs/repeat-*.log` files. The two failed focused fixture runs remain separate. + +The shared 350-case corpus is byte-identical in both repositories. Server tests +consume every wire and stored-value field: 244 accepted cases, plus all 106 +local invalid cases; 104 of those also receive specific legacy-frame NACKs, +while the two deliberately different LONG clamps are covered by D025. The NaN +matrix checks both inputs, all six targets and both row contexts. SF validates +target-native values before persistence and after recovery; byte-identity replay +remains covered by the existing replay suites, not inferred from this test. + +Root verified the successful logs, printed seed values, Surefire's installed-JAR +classpath and project-local temp paths, and final artifact identity. Server +tests use the frozen standalone client without `-Plocal-client`; no reinstall +occurred during server testing. Runtime is Corretto 25.0.4 with Maven 3.9.11. + +- Workspace/installed client JAR SHA-256: + `8e6dfd80195889b30eb1d0a8466f5529a15a242bc19b1d6814b4ff4ee988497e`. +- Client `QwpSchemaBinding.java` SHA-256: + `7ca784dc29c76d1ace2e2ebafef963c5e2d77939c485fdd5c2d04c104b4be53f`. +- Server `QwpSchemaFloatingNumericE2ETest.java` SHA-256: + `178815e8e7dc046f0bcd1e76926047c0bf628d1b791345165e460833dc8389ee`. +- Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD + remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`. + +Tracked production diff hashes in both repositories, the shared buffer/writer/ +Sender hashes, the server schema-control source and all older corpora remain +unchanged. Only the previously untracked client binding has new executable +production changes in this iteration. Both tracked diff checks and explicit +new-file whitespace checks are clean. Dirty worktrees and unrelated server +Rust files are preserved; no commits or pushes were made. + +Root accepts iteration 2.12. Independent SOL review accepted the client code, +corpus and tests, and the final server test contract; the server author also +audited the client conversion independently. Review strengthened evidence without +adding production state or a conversion framework. The performance skill kept +functional coverage separate from unmeasured performance claims. Ordinary +Sender activation, remaining conversion families, inference and producer-owned +row-boundary adoption are still deferred under D022. This is a completed +conversion component, not an activated or release-complete schema-aware Sender. + +## D027 — STRING-to-numeric conversion, characterized before implementation + +Date: 2026-09-10; accepted 2026-09-11. Status: iteration 2.13 accepted after +explicit LONG text overflow approval, review and client/server validation. + +The next bounded slice adds STRING input to BYTE, SHORT, INT, LONG, FLOAT and +DOUBLE in the existing non-owning binding. It directly supports early rejection +of malformed or out-of-range values. Ordinary Sender activation, inference, +row-boundary adoption, numeric formatting and other conversion families remain +outside this increment. D022's integration gate is unchanged. + +Start with real-server characterization using the previously accepted client +artifact. Similar parser implementations do not prove parity: the client reads +Java characters while ingestion reads encoded UTF-8. Explicitly test grammar, +Unicode, malformed surrogates and NUL as well as numeric boundaries. Do not +assume every non-ASCII input is rejected. Server acceptance and stored values +must be established before releasing the production patch. + +Reuse direct target parsers and the existing column append/rollback machinery. +Do not route integer text through DOUBLE, add a converter framework, introduce +another buffer owner or change the legacy Sender/server to fit client behavior. +Preserve duplicate-first checks and existing target/parameter/designated guards. +Any newly discovered contract difference requiring a product choice must be +reported before silently changing acceptance. + +SOL ownership is split between client binding/shared corpus/client tests, +server characterization/E2E/builds, and independent adversarial review. Root +owns architecture, independent evidence checks and this journal. Acceptance +requires shared cases with exact target wire values and SQL results, server +rejection parity, null/sentinel contexts, duplicate handling, partial-row +rollback preserving completed rows, and converted SF recovery. No reflection, +private hooks or test-only production APIs. Maven runs remain serialized per +repository and the installed standalone client is frozen during server gates. + +The performance skill was read with both evidence references. This is a +functional extension, not a production optimization: no bottleneck is claimed +and no measured throughput/latency comparison is available. Its optimization +gate does not establish correctness for this work. Keep implementation scope +small and make no performance claim. Use absolute repository-local test temp +paths because shared `/tmp` is close to capacity; preserve unrelated files. + +### Characterization findings before the production gate + +Source review disproved the initial SHORT simplification: server `parseShort` +accepts only optional minus and ASCII digits, while `parseInt` accepts leading +plus and underscores. A strict grammar check followed by the existing integer +parser and short range check can preserve acceptance without another arithmetic +parser. This is a simplicity choice, not a measured performance improvement. + +Root also reproduced an existing LONG parser overflow through the public +`Numbers.parseLong` API in the accepted 2.12 client artifact: +`"21000000000000000000"` becomes `2553255926290448384`; its negative counterpart +becomes the negative of that value. Values such as unsigned 64-bit maximum +reject, so checking only obvious boundary strings would miss the problem. +Client and server source use the same insufficient wrap-detection arithmetic. +Real-server ingestion confirmation is being run before a product recommendation. + +Independent SOL review agrees that silently adding a checked parser would +violate the current supplied-value parity rule. Root recommends rejecting +mathematically overflowing LONG text in schema mode if ingestion confirms the +behavior, while preserving legacy ingestion and shared parsing utilities. +That would require an explicit compatibility decision; production edits remain +withheld. Preserve other established grammar independently of this arithmetic +decision, including LONG's existing suffix/separator behavior. + +The real-server test confirmed all five supplied wrapping examples (positive +and negative 21e18, positive 25e18, 42e18 and 63e18). It sends legacy VARCHAR +wire values, requires successful write responses and checks the exact stored +LONG values. `testLegacyLongParserWrapCharacterization` passed 1/1 with zero +failures/errors/skips; evidence is in sibling +`questdb/core/target/schema-string-numeric.2.13-server/logs/wrap-characterization.log`. + +The broader diagnostic characterization passed separately in +`logs/characterization2.log`. This records acceptance and stored values rather +than asserting a frozen conformance corpus; it is not converter acceptance. +Its first run, retained as `logs/characterization.log`, stopped on an incorrect +test assumption that every rejection used one status code. Production behavior +was not changed to make the diagnostic pass. + +Root verified both successful logs and the unchanged standalone 2.12 artifact +SHA-256 `8e6dfd80195889b30eb1d0a8466f5529a15a242bc19b1d6814b4ff4ee988497e`. +Client binding, client tracked production diff, server tracked production diff +and server schema-control hashes also remain unchanged. No converter patch or +ordinary Sender activation has landed in 2.13. The user has been asked whether +to approve strict LONG text range rejection in schema mode. This iteration +remains incomplete pending that decision; the dedicated test preserves the +legacy baseline independently of the future choice. + +The server build owner recorded commands and scope in +`core/target/schema-string-numeric.2.13-server/characterization-manifest.md`. +The diagnostic matrix has 366 observations, not a full grammar proof. Root +corrected a summary overstatement: rejection of `1L` and `0x10` does not prove +rejection of floating suffixes or complete hexadecimal floating literals. +Those remain explicit cases for the future shared corpus. All agents have +paused; no client source, tests, resources or build artifacts changed in 2.13. + +### Approved LONG text range exception and resumed implementation + +The user answered "Yes, proceed" to rejecting mathematically overflowing LONG +text in schema mode while leaving legacy behavior unchanged. Revision 27 records +this exception to supplied-value parity. Implementation is now released, not +accepted: all 2.13 converter and regression gates remain to be run. + +Use a private checked LONG text parser in the existing binding if necessary; +do not change shared `Numbers`, ordinary Sender, or server ingestion. Preserve +the existing syntax independently of range checks: no leading plus, existing +underscore/DEL separator rules and terminal `L`/`l`. Arbitrary leading zeros +must not cause false overflow. Exact signed-long endpoints remain accepted; +the minimum is a supplied target sentinel, not a missing STRING input. Do not +route parsed values through the LONG or floating setters, whose source-null +rules apply to different input types. + +Reactivated SOL ownership remains client implementation/corpus/tests, server +E2E/builds, and independent adversarial review. Root will independently check +the checked arithmetic and acceptance evidence. The skill's optimization gate +does not apply cleanly to this requested functional extension; its evidence +discipline is retained without optimization edits or performance claims. Shared +`/tmp` now has roughly 211 MB free; all new test JVMs must use repository-local +temporary directories. No unrelated cleanup, commits, or pushes are authorized. + +### Implementation review and independent checks + +The first candidate adds direct numeric writes and small private SHORT/LONG +parsers to the existing binding. Numeric parse errors share one conversion +boundary; no additional owner, cache, protocol or delivery state is introduced. +SHORT checks its strict syntax before the existing INT parser/range check. +LONG uses checked negative accumulation so both signed-long endpoints remain +representable without overflowing intermediate arithmetic. + +Root compiled the candidate separately from Maven outputs and exercised the +public binding and serialized LONG wire against an independent regex/BigInteger +oracle. All 14,598 deterministic cases passed (2,921 accepted, 11,677 rejected; +seed 2132026), including boundary neighborhoods, known wrapped values, syntax +mutations, Unicode and 10,000 leading zeros. Candidate binding SHA-256: +`724834280cffa6b43d8b229513211c97c9df89d7f55390e4e841dc19511477d1`. +The same diagnostic had failed against the old 2.12 artifact with the expected +missing-conversion error. Final acceptance must rerun against the frozen new +artifact; this isolated compile does not replace Maven or server validation. + +Independent SOL review and root both caught two draft test weaknesses before +acceptance: BOOLEAN text `"2"` is INVALID_VALUE, not an unsupported pair, and +the mutable-input test discarded its first row before observing its converted +value. The author was asked to correct those assertions and format new test +helpers as readable code. Tests must prove their named behavior. + +The legacy NBSP rejection exposed a pre-existing server diagnostic bug, not a +new conversion contract: after numeric parsing fails for `"\u00a01"`, error +logging throws `LogError: Invalid UTF-8` and the response is INTERNAL_ERROR +instead of SCHEMA_MISMATCH. The earlier characterization log records this at +the numeric-error logging path. Keep this server behavior outside the patch; +tests must identify this specific observed response rather than accepting an +arbitrary failure status. Local conversion still rejects the malformed value. + +Runtime checks exposed another load-bearing difference: the server's UTF-8 +memory parsers accept both `f` and `d` suffixes for decimal FLOAT/DOUBLE text, +but the client character parsers accept only their own decimal suffix. The +initial corpus correctly expected server acceptance for FLOAT `"1d"` and DOUBLE +`"1f"`; the binding failed those cases. This was a production compatibility +defect, not a reason to weaken the corpus. Source equality with the server's +character parser had not established parity with its ingestion memory parser. + +The fix remains binding-local: find an opposite numeric suffix before trailing +ASCII whitespace, require a preceding digit or decimal point, then pass a slice +of the original CharSequence to the target parser. It does not copy/rewrite +text, use an exception for a successful conversion, change shared parsers or +add state. The prefix parser validates the remaining grammar. Explicit cases +cover complete hex forms, suffix case, trailing dots, NUL/whitespace, malformed +double suffixes and suffixed NaN/Infinity. No new compatibility exception is +needed; this restores the agreed server behavior. + +Root exercised this candidate binding and serialized floating values against +the actual server UTF-8 memory parsers over 16,628 deterministic inputs (2,806 +accepted, 13,822 rejected; seed 2132027), with zero mismatches. This is a +public-API/parser differential diagnostic, not a real-server ingestion test or +an independent implementation of floating arithmetic. The separate JDK and +BigInteger corpus audit checks 429 cases: all 255 non-null integer acceptance/ +value expectations, 111 accepted floating bit patterns and 6 explicit nulls; +the 57 rejected floating cases only receive structural marker checks there. +Real-server tests own the floating rejection-policy evidence. + +The final candidate corpus has 429 cases (188 accepted, 241 locally invalid), +including the five explicit legacy wrapping exceptions. Client review is +accepted at binding SHA-256 +`a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215` +and corpus SHA-256 +`e1b3728510afd915bd1cf589ff2c2900337614e331ff821d1014cbb6c825c9d1`. +Client focused aggregate passed 53 tests plus 2 packaging checks; canonical +passed 386 tests plus 2 packaging checks. The initial failing fixture/parser +runs remain distinct from the accepted logs. Server ingestion, recovery and +final artifact identity gates are still pending at this point. + +Final source review strengthened the server tests without changing production: +replace the print-based characterization with fixed corpus assertions; require +exactly 236 legacy rejections after the five separately asserted wrap cases; +distinguish the known NBSP logging error from numeric parse/range failures; +check literal sentinel/default values and cross-suffix values through SF. +The encoded identity is compared to the actual DESCRIBE snapshot, not a +synthetic fixture constant. The final focused rerun after that identity check +passed all six tests; the broader server gate passed 312 tests with zero +failures/errors/skips. Three seeded repeats remain before final acceptance. + +Root reran both generated diagnostics against the frozen installed artifact, +not the isolated candidate classes: all 31,226 public-binding/wire cases passed. +The standalone artifact SHA-256 is +`5c1d2a24e6147a066a9fbc2e28e4fdf8faaca1b6baece62af7d1769c02b71144`. +Exact diagnostic commands, scopes and limitations are recorded in +`core/target/schema-string-numeric-2.13/oracle-manifest.md`. + +Reflection: reuse of an existing parser is the right starting point, but +character versus byte entry points can implement different grammar. Test the +actual ingestion path before claiming parity. The two necessary adaptations +stay local and explicit (checked LONG arithmetic and floating suffix handling), +with a strict SHORT grammar check. No conversion registry, compatibility mode, +alternate buffer owner, shared-parser refactor or send-time transformation was +needed. This slice reduces conversion gaps; it does not resolve D022's Sender +row-lifecycle, inference and activation obligations. + +### Final validation and acceptance — 2026-09-11 + +All final gates passed with zero failures, errors or skips: + +- Client focused aggregate: 53 tests plus 2 packaging checks. +- Client canonical: 386 tests plus 2 packaging checks. +- Server focused: 6 tests, rerun after the final schema-identity assertion. +- Server canonical: 312 tests. +- Server repeats: 51 tests each at seeds 1/2, 42/43 and 1234567/7654321. + +The selectors extend the accepted 2.12 gates with +`QwpSchemaBindingStringNumericTest` on the client and +`QwpSchemaStringNumericE2ETest` on the server. Existing conversion, negotiation, +feedback, replay and legacy characterization coverage remains selected. +The shared 429-case corpus is byte-identical in both repositories: all 188 +accepted cases have exact target wire and stored-value checks, all 241 invalid +cases fail locally, 236 also receive the specified legacy rejection, and the +five approved overflow differences have separate exact legacy stored-value +assertions. The Unicode logging failure is specifically asserted, not treated +as permission to accept arbitrary server errors. + +The null matrix distinguishes present parsed INT/LONG sentinels and floating +NaN from explicit bitmap nulls, with and without another omitted row. BYTE and +SHORT missing values are asserted as exact SQL zero. The SF case checks +conversion before persistence and recovered INT 42, FLOAT `"0.1d"` bits +`0x3dcccccd`, and DOUBLE `"-0.0f"` bits `0x8000000000000000`. It does not replace +the existing generic replay byte-identity evidence. + +Client commands/logs: `core/target/schema-string-numeric-2.13/run-manifest.md`, +with final `logs/focused-final.log`, `logs/canonical-final.log` and +`logs/install-final.log`. Server commands/logs: sibling +`questdb/core/target/schema-string-numeric.2.13-server/run-manifest.md`, with +final `logs/focused-final2.log`, `logs/canonical312.log` and all three +`logs/repeat-*.log`. Diagnostic characterization and superseded fixture/parser +runs remain separately recorded. They are not final acceptance evidence. + +Root verified the successful log summaries, final Surefire seeds and absolute +project-local temp path, the installed standalone client on the server test +classpath, and unchanged artifact identity. There was no `-Plocal-client` or +client reinstall during server gates. Runtime: Corretto 25.0.4, Maven 3.9.11. + +- Workspace/installed client JAR SHA-256: + `5c1d2a24e6147a066a9fbc2e28e4fdf8faaca1b6baece62af7d1769c02b71144`. +- Client binding SHA-256: + `a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215`. +- Shared corpus SHA-256: + `e1b3728510afd915bd1cf589ff2c2900337614e331ff821d1014cbb6c825c9d1`. +- Client test SHA-256: + `a475aafdd1c9863d1f04722751a5af3d1296ecac81afdc20d2b9a1c1cb09eb55`. +- Server test SHA-256: + `b5f5f04f118d032cc2da5cea6c7cdbf3de911ec87aa65e8b66703241cb3c3444`. +- Client HEAD: `981bdb02a471f3b290c89b8e78cbc422610e329e`. +- Server HEAD: `12a33d651e51e2682e7a448c8db5168fc72dfad3`. + +Both SOL source reviews accepted the final server test identity; the independent +reviewer also accepted the final client binding, corpus and tests. Root's +generated diagnostics passed again against the final installed artifact. +Tracked production diff hashes in both repositories, shared buffer/writer/ +Sender hashes and server schema-control hash are unchanged from 2.12. The only +new executable production behavior is in the existing untracked binding. +Tracked diff and explicit new-file whitespace checks are clean. No unrelated +files were removed, no shared parser or server production code was changed, +and no commits or pushes were made. + +Root accepts iteration 2.13 as a production-quality conversion component. +The performance skill kept functional verification separate from unmeasured +performance claims; no optimization or hardware-bottleneck claim is made. +Ordinary Sender conversion, remaining conversion families, inference and +automatic row-boundary adoption remain deferred under D022. This is not an +activated or release-complete schema-aware Sender. + +## D028 — Complete missing-conversion inventory — 2026-09-11 + +### Request and scope + +User requested all missing conversions in the design, cross-checked against +server source. This is a documentation-only audit after accepted iteration +2.13, not another implementation iteration. Root owns the document changes; +three SOL agents independently inspected scalar dispatch, complex types and +the public client/binding surface. No production code or tests were changed, +and no builds, commits or pushes were requested. + +### Decisions and findings + +Design revision 28 adds a complete public-input-to-target inventory, including +missing identity setters, parameterized target representations and wire-only +DATE coverage. "Implemented" means the existing binding component, not ordinary +Sender activation. Null acceptance, ignored duplicates and no-ops cannot be +used to claim support for effective non-null values. + +The inventory follows the actual wire cursor, dispatcher and value routine, +not general SQL casting or helper names. STRING input emits VARCHAR; no separate +TYPE_STRING exists. BYTE/SHORT/INT are genuine public input types, not LONG +aliases. SYMBOL is dictionary-backed. Typed decimal and geohash text overloads +parse locally before producing their respective wire types and are not the +same conversion path as stringColumn. + +Missing work includes numeric/date/timestamp/text/decimal cross-pairs, remaining +string parsers, native setters for CHAR/IPv4/LONG256/BINARY, all decimal/geohash/ +array bindings, and numeric/special-type text formatting. Source-specific null +rules remain explicit: approved numeric source-null exceptions do not silently +decide new families, and decimal cursors use bitmap-only null detection rather +than the fixed-width cursor's sentinel inference. + +The source audit found five boundaries that must stay visible until explicitly +resolved: BINARY reaching text parsers; existing-array element/rank validation +gaps despite LONG_ARRAY auto-create rejection; non-ASCII CHAR text corruption +and UTF-16 surrogate representation; designated timestamp fixed-width fallback; +and decimal auto-create scale checks that rely on assertions. These are recorded +as source-proven paths requiring observable tests and a compatibility decision, +not as approved casts, approved behaviour changes or runtime-confirmed failures. + +Corrected the earlier array coverage wording: the server copies elements; it +does not establish LONG/DOUBLE element conversion. Keep missing-table/column +inference, designated row completion and automatic row-boundary adoption apart +from the conversion matrix. Do not add a conversion framework, fallback, +send-time transformation or public DATE setter simply to fill out the table. + +### Source verification and review + +The design contains a linked source map with methods and snapshot line numbers. +Both repositories contain existing uncommitted feature work, so base commits +alone are not presented as the audited implementation: + +- Client base: `981bdb02a471f3b290c89b8e78cbc422610e329e`. +- Server base: `12a33d651e51e2682e7a448c8db5168fc72dfad3`. +- Client binding SHA-256: + `a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215`. +- Server QwpWalAppender SHA-256: + `9d61e060177bae1c10f507f60dc98443fe9ce350d9c0a18790935f58695e5d40`. +- Server WalColumnarRowAppender SHA-256: + `61d80aa814f0674be3aa655f65e8b9570da98b261d6d25c7123bccfcc1cc71b0`. + +Root cross-checked the entire target dispatcher, cursor factory, binding +entrypoints/target selection and the cited parser/formatter boundaries. Root +corrected initial audit assumptions about a nonexistent TYPE_STRING and the +actual UTF-8 sink implementation. Independent draft review corrected the binary +overload list and added decimal-specific sentinel coverage. The public-client +review accepted the corrected inventory without remaining findings; the complex +type review accepted the final corrections and found no missing family or +overbroad target pair. All linked source paths resolve. Tracked diff and explicit +new-document whitespace checks are clean. Client/server tracked production diff +hashes, the binding, ordinary Sender and both audited server appenders are +unchanged from the start of the audit. + +No new runtime coverage is claimed. Iteration 2.13's recorded acceptance remains +the latest test evidence; this audit does not make missing conversions or the +ordinary Sender release-ready. + +## D029 — BINARY target conversion (iteration 2.14) — 2026-09-11 + +Status: accepted as a conversion component; ordinary Sender activation remains deferred. + +### Scope and rationale + +After the conversion audit, the user asked to proceed incrementally. Root chose +BINARY target support: the three existing binary input forms (byte array, +DirectByteSlice, native pointer/length) and stringColumn into BINARY. This is a +bounded family with an existing buffer representation and no new parser. The +unresolved BINARY-to-number/date/etc. parser paths are explicitly excluded, +along with ordinary Sender activation and all other conversion families. + +The existing ColumnBuffer supports BINARY and VARCHAR using the same offsets +and byte storage. Use addBinary for opaque input and addString for existing +UTF-8 encoding; select TYPE_BINARY before append. Add no second buffer owner, +temporary converted byte array, shared parser change or send-time conversion. +The binding remains non-owning and the caller retains row completion/rollback. + +Effective null byte-array/slice arguments are INVALID_VALUE, not implicit nulls. +Native lengths must be 0..Integer.MAX_VALUE, with a nonzero pointer required +only for nonempty input. Reject out-of-range lengths before payload growth or +source-memory access. The documented caller obligation to provide readable native memory +remains; the client cannot validate arbitrary addresses. A zero pointer with +zero length is an empty value. Apply duplicate suppression before argument +validation, as already required for schema-mode setters. + +String null and omission use the target null representation. Empty bytes stay +distinct from null. Root verified that current server MemoryCARW.putBin and +MemoryPARWImpl.putBin preserve zero length, despite stale public Sender Javadoc +saying empty becomes null. That existing documentation discrepancy is recorded, +not used as a reason to copy incorrect behaviour or change ordinary Sender. + +### Validation and ownership plan + +SOL client owner implements the binding, shared literal-byte corpus and public +component tests. SOL server owner independently checks the server contract and +adds real-server discovery/conversion/wire/SQL/recovery tests. A third SOL agent +reviews assumptions and then the implementation adversarially. Root owns the +design/journal, scope decisions, source cross-checks and final acceptance. + +Tests must cover every input form, all byte values, UTF-8 edge cases and lone +surrogates, null versus empty, bitmap boundaries, immediate copying of mutable +or subsequently freed input, ignored invalid duplicates, typed local errors, +rollback retaining completed rows and removing failed-row-only definitions, +reset/clear, exact pinned schema identity and TYPE_BINARY, compression and SF +recovery. Shared vectors carry literal expected bytes; legacy comparisons must +check actual stored bytes. No reflection, private-state inspection or new +test-only production hooks. Oversized native-length tests must reject before +accessing deliberately unusable memory, never allocate multi-gigabyte payloads. + +Maven execution and client installation are serialized. Client focused tests +precede independent review and the extended canonical gate; only the approved +standalone artifact is installed for server focused/canonical/seeded gates. +No server production, shared buffer/writer or ordinary Sender changes are +planned. Previous dirty work and unrelated server parquet files are preserved. + +The performance skill was found under its installed package path after the +catalog path failed. Its measured bottleneck/optimization workflow does not +provide a baseline for this newly supported functional path. Root uses source +copy/allocation inspection and correctness validation here, makes no measured +performance claim, and does not optimize existing shared hot paths. Source +inspection confirms that the existing buffer directly encodes UTF-8 and copies +native input during the call; tests must verify the ownership consequences. + +### Starting source identity + +Base commits remain those recorded in D028. The starting binding SHA-256 is +`a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215`. +Protected tracked production diffs remain: + +- Client: `ab7c4ef0ee366553ec27bbdbe7cf3a517eff44c2fdba15070320fb8a786378af`. +- Server: `cef922f159d0b62fc72cdf330b3fa70f41024568e9de26c17cbaf808c4e422ce`. + +QwpTableBuffer, QwpColumnWriter, ordinary Sender and the two server appenders +match their D027/D028 hashes. The latest accepted tests remain iteration 2.13 +until the final results below are recorded. + +### Adversarial findings and narrow scope adjustment + +The independent corpus review caught a copied-data error: the draft +binary_all_256 vector had only 242 bytes, and both expected fields repeated the +mistake. The corrected vector is exactly 00..ff, with an independent length and +byte-by-byte progression assertion. Draft exact-target enforcement was also +corrected to reuse the binding's existing expected-target helper; BINARY input +must not silently enter VARCHAR merely because the storage layout is shared. + +The requested partial-encoding test then found a real shared-buffer bug. +`focused-second.log` ran 7 tests with one failure: expected byte 1, observed +120 (the injected UTF-8 prefix `x`). On a pre-existing column, addString had +already appended bytes when the CharSequence threw, but had not incremented +size or published the terminal offset. cancelCurrentRow skipped the column +because its size still equaled the completed row count. An explicit truncateTo +at the same size would also be a no-op. Failed-row-only columns were removed +correctly; that case alone would have hidden the existing-column failure. + +Root and both SOL reviewers confirmed that a binding-only repair would need a +new internal checkpoint API or temporary conversion buffer. Root instead +authorizes one narrow shared-buffer correction required to complete this slice: +make addString and both addBinary variable-width appends exception-atomic by +restoring their data and offset positions on RuntimeException/Error. Publish +value and row counts only after success. The guarded region includes encoding/ +copying, checked-offset validation and offset publication/growth; observable +tests inject STRING prefix failures, including after data-buffer growth. Binary +copy/allocation and offset-buffer failures are source-reviewed, not fault-injected. +No new per-column state, callbacks, rollback scan, buffer owner or shared-memory +primitive changes are authorized. Successful encoding and null conventions +remain unchanged; exceptional ordinary Sender cleanup is intentionally repaired. + +This replaces the original no-buffer-edit scope after evidence showed it would +leave the new feature unsafe. Keep the failing test, add VARCHAR/BINARY prefix +failures with exact exception identity, and add ordinary Sender real-server A/C +regression coverage. Do not manufacture OOM/native faults to claim recovery; +arbitrary invalid pointers and fatal process faults remain caller/environment +limits. Other column families' exceptional mutation paths are separate follow-up +work, not silently claimed fixed by this variable-width repair. + +### Final validation and acceptance + +All final gates passed with zero failures, errors or skips: + +- Client focused aggregate, including the shared buffer suite: 109 unit tests + plus 2 packaging checks. +- Client canonical: 394 unit tests plus 2 packaging checks. +- Server focused: 4 real-server tests. +- Server canonical: 325 tests, including the nine pre-existing binary Sender + regressions in addition to the previous schema/compatibility/conversion gates. +- Server repeats: 55 tests each at seeds 1/2, 42/43 and 1234567/7654321. + +The 18 shared cases (4 BINARY, 14 STRING) are byte-identical in both repositories. +Client tests inspect exact target wire bytes, 130-row bitmap/offset boundaries, +all three binary overloads, invalid inputs and duplicates, lifecycle guards, +completed-row retention and failed-row-only definition removal. Mutable inputs +are changed after their setters return, and native input is freed before +encoding; the encoded bytes still match the original values. RuntimeException +and Error prefix failures preserve the original exception object and completed +data, including when UTF-8 capacity grows before the failure. + +Real-server tests obtain live DESCRIBE identities, assert TYPE_BINARY before +sending and compare every stored byte. Separate legacy senders for BINARY and +VARCHAR input preserve legacy inferred-layout rules while checking server +conversion parity. NULL, omission and present empty input remain distinct. +The ordinary Sender VARCHAR and schema-binding BINARY A/failing-B/C regressions +both store exactly A/C after the shared repair. The SF case checks target bytes +before persistence and exact recovered bytes after source memory has been freed. +It does not replace existing generic replay byte-identity evidence. BINARY has +no separate compression encoding; the test checks its unchanged payload in +Gorilla-enabled messages, not a binary compression algorithm. + +Root also ran an independent JDK UTF-8/literal-byte oracle over the shared corpus: +all 18 vectors passed, including an independently checked complete 00..ff +sequence. The diagnostic uses no client or server converter. It is recorded in +`core/target/schema-binary-2.14/CorpusOracle.java`, SHA-256 +`7e769a145326b333c4936875680fd5e27b0f6a443bfa09538e640aa163be740c`. + +Client commands/logs: `core/target/schema-binary-2.14/run-manifest.md`, especially +`logs/focused-final-with-buffer.log`, `logs/canonical-final.log` and +`logs/install-final.log`. Server commands/logs: sibling +`questdb/core/target/schema-binary.2.14-server/run-manifest.md`, with +`logs/focused.log`, `logs/canonical.log` and all three `logs/repeat-*.log`. +Earlier fixture/compiler failures and the decisive pre-repair behavioural +failure remain separately recorded; they are not final acceptance results. + +Final source and artifact SHA-256: + +- Binding: `1b21f16f9312d06f6a3ba741426fd53f16477c0280a2bd01fd196afa1615f94d`. +- Shared buffer: `e6a9451e82058d29094b89472af387c6cb23e986a70a9e24fb32c35d79da7731`. +- Client test: `b5350a7d261611c8b73a343718f20e2ba6d0474ed6d2b21c591da41b46a2dd35`. +- Server test: `ca056571bd8225c217e2975386d18e8eb19e8773d2b6eac67088659ef0108427`. +- Shared corpus: `aba29365c5e148a5bd0eae5f95e8e68dbd441c9715741d1ab4a8d93f5c89813a`. +- Workspace and installed client JAR: + `b9f658d20a0e00a56a6cf287a6d407a849e389d0aa4390890039605bd5fc7394`. + +Root verified final log summaries, the standalone installed artifact on the +server test classpath, explicit final seeds and project-local JVM temp paths. +All runs were serialized; the client was not rebuilt/reinstalled during server +gates, and no local-client profile was used. Runtime: Corretto 25.0.4 and Maven +3.9.11. SOL source review accepted the final binding/buffer/tests/corpus hashes; +the documentation review's fault-coverage wording correction is incorporated. + +The only new executable production changes in this iteration are BINARY target +binding and the authorized string/binary append-position repair. Ordinary Sender +source, column writer, off-heap primitive and server production remain unchanged; +ordinary Sender benefits only from the shared exceptional-cleanup repair. +No array/symbol cleanup rewrite, new buffer owner or measured performance claim +is included. Native copy/allocation fault recovery remains source-reviewed, +not dynamically fault-injected. The five server compatibility boundaries from +D028 remain unresolved and are not activated by this slice. + +Root accepts iteration 2.14. Design revision 29 updates the contract, status and +conversion inventory. Remaining conversion families, inference, row-boundary +adoption and full Sender activation are still required. No unrelated files were +removed and no commits or pushes were made. + +## D030 — Integrate public Sender early; keep completeness as a release gate + +Date: 2026-09-11. Status: accepted for the bounded integration scope; unreleased. + +The user requested integration E2Es, asked root to validate the assumptions, +then authorized implementation. Root and three SOL agents traced current +Sender construction, setters, rollback, batching, schema lookup, negotiation +and the real-server test infrastructure. This changes D022's sequencing, not +the product contract: all conversion families need not precede the first public +Sender E2E. An incomplete automatically activated client remains unreleased. +Unsupported conversions and missing-table/column inference may report explicit +development limitations; no raw-value fallback, feature switch or alternate +Sender is authorized. Existing legacy-mode behavior remains in scope. + +The first driving test uses normal `Sender.fromConfig`, a pre-created UUID +table, valid A, invalid B after another field has been appended, valid C, and +explicit flush. B must fail locally and leave no partial row; SQL must contain +exactly A/C. A complementary public-Sender socket test checks the actual +DESCRIBE, FLAG_SCHEMA, UUID wire representation and pinned table identity. +The current real-server fixture does not capture Sender wire bytes, and SQL +alone cannot distinguish client conversion from the existing server cast. +An absent-confirmation scripted peer proves protocol fallback, not execution +against an older QuestDB release. No actual old-server run is claimed. + +Validated implementation constraints and simplifications: + +- Always request the extension through existing connection factories. The + existing engine/factory requirement latches on first confirmed support, + before any extended bytes; do not add a user option or delay that boundary + until publication. A failed handshake is not legacy confirmation. +- Resolve before the first effective operation of a row, not at `table()` or + flush. Empty buffer allocation is harmless. Use the existing I/O loop and + readiness/lifetime signals; ASYNC must not guess legacy while offline. +- Keep producer-owned row pinning separate from the latest lookup cache. + Replacement connections clear the latest cache, not buffered snapshots. + A cached conversion rejection refreshes once; compare the relevant target + and table incarnation, not metadata-version inequality alone, when deciding + INVALID_VALUE versus SCHEMA_CHANGED. Cancel the whole partial row. +- Reuse owner-backed global-dictionary encoding with `beginSchemaMessage` and + `addSchemaTable`; the component's single-table local-dictionary encoder is + not a replacement for Sender batching. `copySplitMessage` already copies + the schema flag and table body; validate it rather than introducing another + split encoder. Dictionary-only and commit-only frames remain flag-clear. +- `flushPendingRows` publishes data. Schema adoption must retain completed + old-snapshot rows without using a public flush as layout cleanup. Preserve + row/byte accounting, symbol ownership, per-table ordering, reset/close, + split preflight, deferred commits and store-and-forward behavior. Do not + create a general queue framework or transform bytes at send time. +- `atNow()` is a useful first test condition, not a new restricted API + contract. Both row-completion paths must avoid raw writes into schema-bound + buffers; reuse the established timestamp conversion rules when integrating + designated timestamps. No new converter family is justified merely to + broaden this milestone. + +Root owns design/journal and acceptance; SOL owns client implementation and +public socket tests, another SOL owns server E2Es, and an independent SOL +reviews lifecycle, compatibility and complexity. Builds are serialized per +repository, with an explicit installed-client artifact freeze before server +GREEN validation; the initial real-server RED uses the accepted 2.14 artifact. +Use project-local JVM temp directories because shared `/tmp` has only about +311 MiB free. No destructive cleanup or commit is authorized by this increment. + +The performance skill is used for the performance-sensitive Sender path. +This is authorized functional integration, not a production optimization or +hardware-bottleneck claim. Its pre-optimization measurement gate is not claimed +to have passed; do not bundle speculative optimization into this work. Review +added allocation, locking and repeated scans separately from correctness, and +do not present source inspection as a throughput or latency measurement. + +Validation evidence, final scope and review outcomes will be appended here +after the actual test runs. A first passing E2E alone is not iteration acceptance. + +### First failing public-path tests + +The client socket test failed before production changes: malformed UUID text +was accepted by ordinary Sender and no DESCRIBE was sent. This is the intended +missing integration, not a converter failure. Evidence: +`core/target/schema-sender-2.15/logs/focused-red.log` (one test, one failure). + +The real-server test against the frozen 2.14 client also failed on the public +path: after a string UUID input bound legacy VARCHAR, a duplicate typed UUID +setter reported a local wire-type mismatch. Schema mode must instead keep the +target UUID binding and ignore the duplicate before conversion. Evidence: +sibling `questdb/core/target/schema-sender.2.15-server/logs/` +`red-installed-2.14-schema.log`. Superseded fixture compilation/configuration +errors are recorded by the server owner and are not counted as behavioral RED. + +Root rejected a review suggestion to keep a successfully negotiated legacy +producer permanently in legacy mode after a supporting-server reconnect. That +would silently bypass the agreed one-way upgrade. Pending legacy blocks must +keep their old framing, while the next effective row adopts schema mode. +Uniform schema generations can share a schema frame; only mixed legacy/schema +blocks need separate frames. Any rare mixed-family path must size-check all +groups before its first publish and preserve deferred commit and symbol state. + +### First real-server GREEN, before full integration acceptance + +The first public socket test passed after negotiation/string routing was wired. +A deliberately intermediate standalone client artifact was then installed and +frozen for the real-server test, SHA-256 +`8180dc92d99122013ce3cfcc2d5a4c84781e25b1d3aa58ad0871895a8830a0ed`. +`QwpSchemaSenderE2ETest` passed 1/1 against that artifact. It uses normal +`Sender.fromConfig`, pre-created schema, string/UUID inputs into the same UUID +target, target-first duplicate handling, local cancellation of invalid B, +same-table C without another `table()`, `atNow`, explicit flush/ACK and exact +A/C SQL. No manual binding/encoding occurs in the test's write path. + +Server evidence: `core/target/schema-sender.2.15-server/logs/` +`intermediate-first-green-final.log`. The prior run reached its SQL assertion +but incorrectly nested a query memory check inside a live-server context; +the final query uses the fixture's normal outer memory-leak check. The installed +artifact hash was unchanged through both runs. Client work continued without +reinstallation during that gate; the server owner released the artifact only +after stopping Maven. This checkpoint does not validate unfinished setter, +refresh, generation, split or recovery paths and is not iteration acceptance. + +### Rejection recovery clarification + +An implemented input family refreshes once on cached INVALID_VALUE or +UNSUPPORTED_FEATURE, including a column that was absent from the cached schema +but has since been added. Compare table incarnation and the relevant column's +existence/type/parameters. An unchanged target preserves the original reason; +an unrelated version change does not become SCHEMA_CHANGED. An intrinsically +unimplemented input family fails directly. Authorization/availability failures +are not conversion retries. Freshness is per setter: a lookup by an earlier +setter in the row does not make a later setter's rejection freshly validated. +This sharpens the existing error contract without adding setter replay or +retaining original values. A sender-reused lookup-result holder may report +cache/fresh origin without allocating a result per row or using an exception +for an ordinary cache miss. + +### Candidate 2 integration validation + +The second installed candidate, SHA-256 +`78cb193a394dbff4136c45118bc24113fd0698180f4598e943e4f8cb946064e7`, +passed all eight real-server `QwpSchemaSenderE2ETest` cases. These add exact +micro/nano designated timestamps, primitive and Instant inputs, failed timestamp +rollback, cold setter-free `atNow`, relevant versus unrelated DDL with buffered +A, repeated cached MISSING, denied lookup recovery, observable auto-flush, +reset discard and close-only publication. The test's typed UUID uses different +low/high limbs, so reversed argument routing cannot pass. Auto-flush is observed +through ACK 0 and SQL before any explicit flush; no explicit flush can mask it. +Cold `atNow` uses a separate fresh Sender. Timestamp tests also assert total +cardinality, not only filtered values. Reset currently covers the active +generation; do not claim its retired-generation case from this test alone. + +Server log: `core/target/schema-sender.2.15-server/logs/candidate-2-eight.log`. +The installed JAR remained byte-identical during the run. Root independently +checked the log and both workspace/installed JAR hashes. Client focused +consolidation passed 74 tests (`logs/focused-current-3.log`); comprehensive +socket tests and canonical regressions are still pending. Exact candidate +source hashes and commands are in both repositories' 2.15 run manifests. +Root verified Maven 3.9.11, Corretto 25.0.4 and project-local JVM temp storage +against the running Maven and Surefire XML, not only the prior manifest. + +The rare mixed legacy/schema batch uses one frame per table block and a +two-pass size-check/publish loop, reusing the existing encoder. Uniform schema +blocks retain combined/split batching. There is no new queue framework or +send-time transformation. The old split-path at-least-once rule remains: +partial publication failures retain all source rows and retries may duplicate +the published prefix. A review request to reset only published groups was +rejected because it would introduce new bookkeeping and a stronger guarantee. +Size rejection must still publish nothing; deferred commit debt must remain +visible to recovery and close. + +### Candidate 2 broad regression and invariant review + +The broader prior-325 plus new-eight server selection ran 333 tests: 10 failures +and 48 errors, not a passing regression suite. The new eight tests passed in +that run and in three seeded repeats. The separate protocol/discovery/identity/ +feedback baseline passed 214 tests. Existing public-Sender tests now negotiate +schema mode: missing inference, unimplemented conversions and deliberately +different schema-mode semantics expose remaining release work. Some old +conversion fixtures also write through the existing `@TestOnly` buffer seam; +those are not evidence about the supported public row API. Preserve the failing +evidence and classify it; do not silently rewrite legacy expectations to claim +a clean release. The exact failures are recorded in the server run directory's +`logs/broad-failure-lines.txt`. + +Root rejected a mock-only proposed fix that changed a target type while keeping +the same `(tableId, metadataVersion)`. The reviewer could identify no coherent +server path producing that response and retracted the finding. The pair is an +immutable encoding identity, relied on by both buffered rows and replay. Adding +target-change adoption for an unchanged identity would conceal a broken +protocol assumption. The narrow accommodation and impossible mock regression +were removed; real version-changing DDL remains covered. + +Independent production review found no confirmed lifecycle or delivery defect +in candidate 2. Root continues to review observable test sensitivity: exact +wire types and asymmetric UUID limbs, both pinned identities across generations, +no manual cancellation masking automatic rollback, and observable reconnect +confirmation instead of timing assumptions. These checks precede final +iteration acceptance; they do not remove the release blockers above. + +The independent follow-up classified all 58 failures: 38 missing conversions +(22 decimal-related), four missing-inference/lookup-versus-argument-validation +cases, seven documented schema-mode semantic differences, four obsolete +direct-buffer `@TestOnly` fixtures, and five old server-error/message assertions +now receiving local typed rejections. The named LONG/designated timestamp +failures belong to the direct-buffer fixtures, not the public timestamp setters. +This diagnostic selection is not an exhaustive count of missing input families. + +Root also rejected waiting through every observed reconnect gap for a sender +that had already confirmed legacy mode. That would unnecessarily stop legacy +store-and-forward buffering during an outage. Only a never-confirmed sender +must wait to select its initial mode. Once support is confirmed it stays +mandatory; before that, confirmed legacy rows remain valid even offline. The +upgrade test must observe the installed supporting connection before asserting +the next row's mode, rather than use a server-start timestamp to change the +production contract. + +The encoder now rejects table counts outside the wire header's unsigned-16-bit +range before writing a header. Sender also rejects more than 65,535 pending +nonempty blocks before publication, rather than truncating a combined header. +This is conservative: separately grouped frames could theoretically carry a +larger logical batch. Supporting that would require a further batching change; +it is not part of this increment and is unrelated to the million-entry schema +cache ceiling. Boundary tests cover zero, 65,535, negative and overflow counts. + +### Final client gate and artifact freeze + +The final focused selection passed 142 tests plus two packaging checks. The +canonical selection passed 410 tests plus two packaging checks, all with zero +failures/errors/skips. This extends the prior 394-test client selection with +15 public-Sender socket tests and the encoder header-range test; it is not a +claim that every repository test was run. Logs: client run directory +`logs/focused-final.log` and `logs/canonical-final.log`. + +The 15 socket cases include exact target UUID bytes, asymmetric limbs, automatic +partial-row rollback, cached missing tables, no-op and duplicate behavior, +legacy VARCHAR bytes with no describe, both pinned identities across an +unrelated schema update, exact cap-split payloads, retired-generation reset, +startup waiting, actual negotiation deadline/retry and interrupt preservation. +The mixed upgrade sequence completes legacy A, starts legacy B while offline, +observes the installed supporting connection via server PING/client PONG, +finishes B unchanged, then buffers schema C. Flush emits exact legacy A/B +followed by schema C with the correct deferred/final commit flags. No sleep, +reflection or production testing bypass establishes the transition. + +Several old client fixtures used an unconnected Sender with injected connection +state. They now connect to a real non-confirming socket peer through public +construction, retaining the original cap/autoflush/input assertions. Existing +unrelated testing seams were not removed and no new ones were added. The mock +server's additions are ordinary wire PING/PONG and negotiated schema replies. + +The final standalone install succeeded with main JAR SHA-256 +`b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`; +the tests JAR is +`48e9665b4bc5b05f4134899c6e301d6e834a067152dfac88b2d4c4f527e786f8`. +Root independently verified workspace/installed main JAR equality, final +source/test hashes, the 410+2 result and Surefire's Corretto 25.0.4/project-local +temp settings. Exact commands and hashes are in +`core/target/schema-sender-2.15/run-manifest.md`. The artifact is frozen for final +server validation; local installation is not publication or release. + +Separate source reviews found no confirmed production blocker. Root reviewed +test sensitivity independently of the test author. Candidate-2 executable +Sender behavior was retained; the later functional production delta is the +encoder count guard. First-use gating, generation replacement and mixed-family +batching reuse the existing I/O loop, table buffers and encoder. The performance +skill guided allocation/ownership scrutiny and avoidance of speculative +optimization, not measured throughput claims. Lookup/cache overhead has not +been benchmarked. The lookup deadline remains fixed at 1000 ms; the proposed +configuration key is not implemented in this slice. + +### Final server gate, acceptance and remaining work + +Against the frozen final standalone JAR, the eight public-Sender E2Es plus +214 protocol/discovery/identity/feedback/transport tests passed 222/222 with no +failures/errors/skips. Each of three public-Sender repeats also passed 8/8, +with seeds `1/2`, `42/43` and `1234567/7654321`. These are repeated executions +of the same eight tests, not 24 additional scenarios. Server evidence is in +`../questdb/core/target/schema-sender.2.15-server/run-manifest.md`, +`logs/final-222.log` and `logs/final-repeat-*.log` within that run directory. + +The server E2E source hash is +`6e3c29d854e7986170224a9ef8e4cb80ea3e440d3e847ed5d2c8712775501a64`. +Before/after client artifact hashes match. Root independently checked the +222-test result, the final eight-test XML, its standalone installed-client +classpath and absolute project-local temp path. No `-Plocal-client` substitution +was used. Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; +server HEAD remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with +preserved working-tree changes. Final diff whitespace checks passed in both +repositories. No commits or pushes were made. + +Root accepts iteration 2.15 for its stated scope: ordinary Sender automatically +uses the existing schema machinery, fails unsupported writes explicitly, keeps +row/block ownership and delivery semantics, and has working public-path E2E +and exact-wire coverage. Separate SOL reviews found no unresolved blocker in +that scope. This acceptance does not turn the broad 333-test diagnostic into +a passing regression suite or authorize release of the incomplete client. + +The remaining release work is still explicit: + +- Implement the audited conversion families and missing-table/column inference. + The broad failures are useful evidence of these gaps, not tests to disable. +- Separate old-wire compatibility expectations from approved schema-mode + differences in the final regression matrix. Actual old-server distribution + compatibility was not exercised by this increment's scripted peer. +- Extend the ordinary Sender E2E through file-backed SF/process restart; the + existing component/engine recovery tests were retained, but the new public + Sender tests do not claim that additional scenario. +- Finish proposed configuration wiring and validate performance before making + latency or throughput claims. No new conversion framework, user opt-out or + send-time transformation is implied by any of this work. + +Reflection: the integration exposed useful contract and fixture errors sooner +than another converter-only increment would have. The most valuable +simplifications were keeping one connection owner, one buffer owner and the +existing encoder/split path, plus refusing mock-driven changes to immutable +schema identities or legacy outage behavior. Continue with bounded conversion +families exercised through this now-working public path; do not return to a +separate final integration phase. + +## D031 — Public Sender creates and recovers its own backlog (iteration 2.16) + +Date: 2026-09-11. Status: accepted for the bounded public-Sender process-restart +test scope; the complete feature remains unreleased. + +After accepting 2.15, the user asked for next-step reasoning and approved the +recommendation to validate public-Sender restart before another conversion +family. Existing recovery tests construct frames and append them directly to +`CursorSendEngine`, then use Sender to recover. That proves the replay component, +not the complete owner-level path from public setters through file-backed +publication and a producer-process crash. Prioritization follows that risk, +not the number of broad-suite failures attributed to DECIMAL. + +The bounded implementation is a real-server E2E, using the unchanged accepted +2.15 standalone client JAR: +`b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`. +No converter, protocol, recovery API or production hook is planned. A confirmed +production defect must be reviewed before a narrowly necessary fix is made. + +Acceptance sequence: + +1. A child JVM creates normal `Sender.fromConfig` with `sf_dir` and `sender_id`. + Public writes produce valid UUID/SYMBOL A, partially invalid B, then valid C. + B fails locally and is not manually cancelled by the test. +2. A test-only network gate forwards WebSocket upgrade and schema discovery, + but captures and withholds table-bearing QWP data. Wait for both the child's + explicit post-`flushAndGetSequence` publication marker and the captured data. + Verify SQL is empty before stopping the process. +3. Force-stop the child without calling Sender.close, require a bounded process + exit, and close/join its old sockets and relay threads. Start a fresh Sender + with the same public directory/identity configuration. +4. Drain to the real server, compare captured unmasked QWP data before/after + restart, and assert exact A/C values, no B, symbol integrity and ACK progress. + Reopen after acknowledged recovery to check that no rows replay again. +5. Extend the same fixture to A and C encoded under different schema identities, + using the already-tested UUID-to-VARCHAR DDL/whole-row retry sequence. + +Source-backed limits and simplifications: + +- `flushAndGetSequence` returns after producer-owned `appendBlocking` publishes + the mmap-backed frame; it does not wait for a server ACK or imply fsync. + The current MEMORY durability policy survives a JVM crash through the OS page + cache. This is a process-crash test, not a host crash or power-loss test. + Reserved FLUSH/APPEND durability settings remain unsupported and unchanged. +- Exact A/C cardinality is valid because the gate proves zero initial delivery. + This does not strengthen the general at-least-once retry contract. No dedup + table should hide an accidental first delivery or duplicate replay here. +- Compare decoded QWP payloads, not WebSocket masks. Permit extra dictionary + catch-up/control frames during recovery. Test the original table-bearing bytes + and pinned identities; never reconstruct persisted files or inspect private + ACK/segment bookkeeping. +- UUID plus SYMBOL exercises symbol replay integrity. It is not proof that a + dictionary side file alone can reconstruct references after all introducing + frames have been trimmed; that stronger scenario is outside the initial test. +- Reuse the real server fixture and public protocol constants. A small gate is + justified because the existing transparent ACK tee cannot selectively pass + discovery and withhold data. Do not build a general proxy/process framework. + All barriers, process joins and socket/thread cleanup must be bounded. + +Root owns architecture, source validation, design/journal and acceptance. SOL +owns the new server test and serialized server Maven runs; a separate SOL +performs adversarial source/test review. The client artifact stays frozen. +Root verified Maven 3.9.11, Corretto 25.0.4 and artifact hashes. Shared `/tmp` +still has about 311 MiB free: parent and child JVMs must use explicit project-local +temp directories under the 2.16 run directory. Preserve both dirty worktrees; +no commits, pushes or broad cleanup are authorized. + +### First public process-restart checkpoint + +The first one-generation test passed 1/1 against the frozen 2.15 artifact: +server run log `logs/focused-fourth.log`. A child JVM used ordinary Sender to +create file-backed UUID/SYMBOL A/C rows after rejecting partial B. The parent +observed public local publication and a withheld schema data frame, verified +SQL count zero, forcibly stopped the child with nonzero exit, and reopened the +same configuration through a new Sender. ACK progress and exact A/C SQL passed. +At this early checkpoint, exact replay-byte comparison, post-ACK reopen and +the two-generation extension remained pending; it was not full 2.16 acceptance. + +The first three runs exposed fixture errors, not a production recovery defect: +an incorrect hard-coded control flag withheld DESCRIBE, the replacement schema +flag referenced the wrong constants class, and `auto_flush_rows=0` is invalid +configuration syntax. Logs `focused-initial.log`, `focused-second.log` and +`focused-third.log` retain these failures. The final fixture uses public +protocol constants and the already-working 2.15 auto-flush settings. + +Independent review also required explicit child temp storage, one tracked +stdout reader instead of competing/dropping readers, controlled complete masked +WebSocket messages, bounded process exit and relay/reply-thread cleanup. These +are real subprocess/network observations, not injected Sender state. No +production change was required for the first passing checkpoint. + +### Final implementation, review and acceptance + +`QwpSchemaSenderRecoveryE2ETest` now contains two real-server cases. The first +recovers UUID/SYMBOL A/C after local `INVALID_VALUE` rejects partial B, then +reopens the acknowledged slot and verifies no rows replay again. The second +holds A while the server drops UUID `id` and adds VARCHAR `id`; B encounters +`SCHEMA_CHANGED` and is cancelled, then C uses the new schema. The public server +wire cursor verifies two one-row blocks with the same table identity, different +metadata versions and native UUID/VARCHAR types. Both cases require local +publication, captured table-bearing data, SQL count zero before force-stop, +nonzero child exit, acknowledged recovery, identical unmasked QWP payloads and +exact final A/C values. No reflection, private persistence inspection or +manually appended replay frames were introduced. + +The independent SOL review and root review found fixture defects worth fixing +before acceptance: + +- Joining the relay while holding its socket-registration monitor could stall + cleanup. Shutdown marks/closes sockets under the monitor, then joins outside. +- An extra or malformed frame arriving after the initial capture assertion could + set a relay failure that was never checked. Health is now checked after both + relay threads terminate, so a rejected extra frame cannot hide a false pass. +- Exceptional cleanup must explicitly verify the forced child's termination; + interrupted relay cleanup must fail, not silently skip its checks. + +All were corrected without production changes. Final source SHA-256: +`9183f38aebccfac35bd2658aee2ad861d7dcec19adb8bff571ca3ea04356488d`. +Independent adversarial review accepted that source, and root independently +checked the assertions, final logs, artifact hash and whitespace checks. + +Final evidence is under server +`core/target/schema-sender-recovery.2.16-server/`; `run-manifest.md` records +commands, runtime, artifact provenance and superseded development runs: + +- `logs/final-230-rerun.log`: 230 tests, zero failures/errors/skips. This combines + the accepted 2.15 server selection of 222 tests, the two new public recovery + cases and six existing timestamp/text/string-numeric/floating/boolean/binary + component replay methods. +- `logs/repeat-1-2-final.log`, `logs/repeat-42-43-final.log` and + `logs/repeat-1234567-7654321-final.log`: both new cases passed each seed pair. + Earlier green logs predate the final cleanup corrections and are checkpoints, + not the final acceptance evidence. +- The workspace and installed standalone client JAR still have SHA-256 + `b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`. + The earlier 410 client tests and two packaging checks were not rerun in this + test-only increment; they remain 2.15 evidence, not new 2.16 results. +- Maven 3.9.11 / Corretto 25.0.4; parent and child JVMs use project-local temp + storage. Server Maven runs were serialized. Both worktrees pass + `git diff --check`; no producer or Maven processes survived the final runs. + No commits or pushes were made. + +Reflection: the existing production ownership and replay paths survived the +real process boundary without another recovery layer. The added complexity is +confined to a bounded test-local child process and data gate. Keep this public +path as the integration baseline for future conversion families instead of +building another parallel Sender or send-time conversion path. + +Acceptance does not erase D030's 58 classified broad-suite failures or prove +actual-old-server distribution compatibility. Conversion/inference coverage, +configuration wiring and measured performance remain release work. This test +also deliberately leaves power loss and dictionary-side-file-only recovery +after introducing frames are trimmed outside its claim. + +## D032 — Resolve failures without losing the compatibility oracle (iteration 2.17) + +Date: 2026-09-11. Status: first bounded legacy-comparison repair accepted; +the remaining 56 broad-suite failures are unresolved. + +The user authorized exploring and resolving the failures after 2.16. Root +assigned SOL a frozen-client broad rerun, another SOL independent prioritization +and compatibility review, and a third SOL the next functional inference design. +Root owns the source cross-checks, scope, journal and acceptance. The installed +2.15 standalone client remains frozen at SHA-256 +`b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`. + +### Reproduction and corrected classification + +The original 333-test selection ran again. The first result was 12 failures +and 48 errors, but the apparent increase from 58 to 60 was not two new schema +defects. The invalid floating and string numeric corpus tests already send +unflagged legacy frames; their new INTERNAL_ERROR responses contain +`No space left [size=1048576]`. Root rejected the initial classification of +those two as schema/legacy semantic differences after reading their actual +write path and the server error immediately before each failed assertion. + +Evidence: server `core/target/schema-failures.2.17-server/logs/baseline.log` +and `failure-summary.txt`. The working classification is: + +| Group | Baseline count | Resolution rule | +| --- | ---: | --- | +| Missing conversions | 38 | Implement bounded families with source, wire and SQL parity checks | +| Missing inference / validation order | 4 | Restore confirmed-missing inference, preserving lookup failures and server policy | +| Approved schema/legacy differences | 7 | Keep distinct, explicit tests of both contracts | +| Old direct-buffer / wrong-frame fixtures | 4 | Use real legacy wire tests; do not pretend they are public Sender writes | +| Old error shape/message expectations | 5 | Assert local typed failure and row outcome, retaining server rejection coverage | +| Disk-space failures in raw invalid corpora | 2 | Correct runtime storage; do not loosen expected rejection status | + +The first implementation group repairs the Boolean, timestamp, floating-numeric +and string-numeric legacy comparisons, including false comparisons that were +passing. Two SOL authors own disjoint pairs of server test files; the third +reviews all four and root independently checks load-bearing findings. Keep +the schema binding, rollback and public Sender recovery branches unchanged. +Use existing non-requesting WebSocket clients, source-native table buffers and +the existing unflagged encoder. No alternate Sender, schema-disable switch, +production hook, reflection or general test framework is authorized by this +group. Server Maven runs remain serialized. + +### Runtime and review corrections + +Maven `-Djava.io.tmpdir` alone produced a reassuring Surefire XML property but +did not establish the JVM's actual temporary-file root. Root inspected the +running JDK's `File.TempDirectory` and `TempFileHelper` bytecode: both use +`StaticProperty.javaIoTmpDir()`, which retains the startup value. Set the +absolute project-local directory in the fork's actual `argLine`, not only a +later system property. Shared `/tmp` has only about 311 MiB available; the +project filesystem has about 26 GiB. No unrelated temporary files are deleted. +On the corrected focused run, root verified both the actual Surefire Java +launch command and its open WAL/metadata file descriptors under the explicit +project-local `tmp/focused4-argline/junit.../dbRoot` directory, not merely XML. + +This also corrects D031's overconfident parent-temp provenance claim: its +Surefire XML property did not prove the parent test JVM's actual temporary-file +root. Its child used an explicit JVM argument. The passing 2.16 test outcomes +remain evidence; the stronger storage-location claim does not. + +Review findings must survive source checking. Root rejected a proposed broad +timestamp-corpus repair after verifying that `timestamp-units.tsv` contains +only 26 MICROS and 22 NANOS cases (all four invalid cases are MICROS). The +reviewer had conflated it with the separate broader `timestamp-inputs.tsv` +corpus and retracted the finding. No seven-unit normalization framework is +needed to repair that two-unit legacy-wire test. A separate renamed raw-wire +characterization must not claim to execute public Sender's `Instant` or +larger-unit normalization merely because it injects their known output values. + +### Next functional group: confirmed-missing inference + +After the comparison tests are reliable, prioritize the common auto-create path +over choosing DECIMAL merely because it accounts for 22 failures. The reviewed +minimal mechanism is inference within the existing non-owning binding and +caller-owned table buffer, not synthetic server schemas or a second registry. + +- A MISSING response is authenticated absence, not permission to create a + table: `QwpSchemaControl.snapshot` returns MISSING before table authorization. + Server write-time authorization and auto-create policy remain authoritative. + DENIED, UNAVAILABLE and TOO_LARGE must never become inferred schemas; keep + their existing distinct failure/retry semantics. +- Preserve the coordinator's existing result filter: only KNOWN/MISSING reach + the producer; other responses already throw typed errors before identity + reuse. Root's concern that a non-known `(-1,-1)` response might reuse a missing + binding was disproved by tracing `completeResponse` and `resolve`. Test that + invariant with inferred rows, rather than adding a redundant Sender branch. +- Infer established native targets only for already-supported setter families. + Preserve existing timestamp inference: NANOS selects nano precision; other + primitive units and `Instant` select micros. For a known target, its actual + precision still wins. This does not introduce new conversion families. +- Buffer column definitions pin inferred types and existing rollback removes + failed-only definitions. A narrow inferred-column lookup must suppress a + same-row duplicate before checking its setter type; the existing generic + lookup checks type first. Do not globally change legacy duplicate semantics. +- Missing tables keep schema framing with unknown identity `(-1,-1)`; missing + columns on known tables retain the known identity. ACK metadata starts a new + generation through the existing adoption path, never relabeling buffered bytes. +- Infer an unnamed designated timestamp only for a missing table. A known + table without one must continue to reject explicit `at(...)`; `atNow()` + retains server-assigned time. Test all these cases through public Sender. + +This is a source-reviewed next-step recommendation, not implemented inference. + +### First repair: controlled broad-suite result + +The final compiled four-class focused run passed 25/25 with the corrected +JVM-startup temp setting (`logs/focused4-acceptance-2.log`). The identical broad +333-test selection then reported 9 failures and 47 errors, zero skips +(`logs/broad-acceptance.log`). Its 56 failing method names are exactly the first +run's set minus these four: + +- Boolean schema/legacy omission comparison: repaired legacy producer path. +- Timestamp normalization characterization: repaired raw legacy producer path + and narrowed the method name/claim to what it actually executes. +- Invalid floating corpus and invalid string-numeric corpus: their original + rejection assertions pass after correcting temporary storage; these were + environment failures, not repaired production conversion defects. + +Thus two of the original 58 contract/fixture failures are resolved, not four. +The remaining 56 consist of 38 missing conversions, four inference/order cases, +five remaining schema/legacy expectation differences, four old direct-buffer +fixtures and five error-shape assertions. The accepted floating/string and +other Boolean comparisons were also corrected even though they had been green: +they now observe real legacy source bytes instead of comparing local schema +conversion with itself. + +Stronger raw-wire assertions required respecting the public cursor contract: +reading a FLOAT through `getDouble()` quiets a signaling NaN, so inspect the +original fixed-width payload after validating its count, width and absent null +bitmap. VARCHAR's null value accessor is an empty flyweight; test the structural +null flag separately, then compare all non-null bytes, including present empty +strings. Legacy BOOLEAN omission must use its actual non-nullable source layout +to preserve false, rather than introducing a null bitmap. Existing SQL values +and rejection statuses were retained. Development logs preserve the temporary +directory, cursor API/exception declaration and assertion mistakes; only the +final compiled source is eligible for the remaining regression gates. + +### Final acceptance and next boundary + +All final gates used the same four test sources and the unchanged installed +client JAR. Evidence lives under server +`core/target/schema-failures.2.17-server/`; `run-manifest.md` records exact +commands, runtime and superseded runs: + +- `logs/focused4-acceptance-2.log`: all 25 affected tests passed. +- `logs/broad-acceptance.log`: the unchanged 333-test selection has 9 failures, + 47 errors and zero skips. The exact method-set comparison shows only the + four removals described above and no newly failing methods. +- `logs/final-251.log`: 251 tests passed, zero failures/errors/skips. This + retains the previous protocol, negotiation, feedback and public Sender/crash + recovery selection, including text/binary replay, and runs all four repaired + conversion-comparison classes in full. +- `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 25 affected tests passed each seed pair. + These are repeated executions, not 75 additional scenarios. +- `logs/final-before.sha256` and `logs/final-after.sha256` record the frozen + artifact and test-source identities. Final test source SHA-256 values: + Boolean `d4041c6530bb0b980e9b96657b08e12fbad95da34bf223a1eb6b1cebd9da7a53`; + timestamp `7b897ea71a5d710a3cdaa570631c76319439f4905741469f85af8255f778c1d8`; + floating `fe7cc286490c9de1de49a7fe3247962cb387f8525dc74c4ae6e272f15f874aed`; + string-numeric `68d9fc992b6c07242acb306acc7df1356610ce7b008a3544df0abd6147f0e269`. + +The independent SOL reviewer accepted the final wire/null/NaN checks; root +independently verified those contracts, logs and source hashes. Client tests +were not rerun or repackaged in this server-test-only increment. No production +source, conversion policy, fixture opt-out or testing hook was added. Existing +dirty work was preserved; no commit or push was made. + +Reflection: passing SQL is not an independent conversion oracle when both +writers now run the same client conversion. Inspect the emitted source bytes +and keep client conversion and server conversion as separate tests. Conversely, +an increased failure count is not proof of another feature regression: inspect +the server cause and actual runtime configuration. Preserve these corrections +before moving on to the source-reviewed auto-create inference implementation. +The complete feature is still unreleased; this iteration does not claim to +resolve all failures, prove old-distribution compatibility or measure performance. + +## D033 — Confirmed-missing inference using the existing buffer (iteration 2.18) + +Date: 2026-09-11. Status: accepted for the bounded inference scope below; +the remaining 54 broad-suite failures are unresolved. + +Continue with D032's functional priority: permit automatic table and column +creation for setter families whose schema-mode conversion is already supported. +The user reiterated that simplicity is a requirement: read enough surrounding +code to justify each change, and avoid speculative defensive checks. + +Root rechecked the binding, producer row-boundary adoption, coordinator result +filter and server schema discovery/write paths. The buffer already owns column +definitions and row rollback; inference should use that state. No synthetic +server schema, inferred-column registry, protocol change, opt-out or send-time +conversion is needed. MISSING permits inference, not creation authorization. +The existing server remains responsible for authorization and auto-create policy. + +SOL owns the narrow client implementation and client tests; another SOL owns +public-Sender server tests; a third independently reviews contracts and unnecessary +complexity. Root owns integration, serialized builds, source checks and acceptance. +Preserve earlier dirty work and keep additional conversion families out of scope. +Required observations include source-native inferred bytes, known/unknown schema +identity, duplicate handling, failed-row rollback, timestamp precision, ACK-driven +adoption and rejection by server policy. No release or performance claim follows +from this increment alone. + +### Source-review refinement: inferred type conflicts must refresh + +The generic buffer rejects a later row's different wire type with an ordinary +`LineSenderException`. Reusing that error unchanged in inference would bypass +Sender's existing schema-rejection refresh: a cached MISSING result could keep +rejecting a string setter after another writer creates a compatible VARCHAR +column. Both root and the independent reviewer traced this concrete path. + +Keep the generic legacy lookup unchanged. In the binding's missing-column path, +look up the existing buffer column, ignore an already-written value in the same +row, then report a conflicting inferred type as `UNSUPPORTED_FEATURE`. This +uses the existing one-refresh/whole-row-retry path. Do not catch arbitrary buffer +exceptions, parse their messages or add a second retry loop. The name-only lookup +can expose the existing column's type and size; no result enum, callback or new +state is needed. Test unchanged MISSING, newly created server targets and failed +refresh, with completed bytes retained in every case. + +Root also rejected an impossible draft test fixture: current server SQL only +enables WAL on partitioned tables, which require a designated timestamp, and +QWP rejects writes to non-WAL tables (`SqlParser` and `QwpTudCache`). A real +known table without a designated timestamp can prove local `at(...)` rejection +and recovery to another writable table. It cannot prove successful QWP storage +via `atNow()`. Keep local row-completion behavior and server writability separate; +do not change production or weaken server checks to make the fixture work. + +### Initial runtime and simplicity corrections + +The first focused client run compiled and executed 140 tests: five failures, +zero errors. All five new wire assertions assumed an unknown identity was +encoded as discriminator 1 followed by literal `(-1,-1)`. The existing writer +uses discriminator 0 with no pair bytes; discriminator 1 introduces a known +pair. Correct the test readers, not the encoder. Static review missed this +and its claim of proven byte preservation was withdrawn pending a green run. + +Root also removed a proposed duplicate-designated-timestamp guard and its two +nullable-result branches. Public `at()` already commits the row; repeating a +designated setter within one uncommitted row was only a component fixture +scenario, and the known-schema path did not support that new behavior anyway. +Keep across-row inferred precision conflicts typed and refreshable, without +inventing an additional within-row designated timestamp contract. + +After those corrections, the focused client gate passed 140/140. The first +broader gate ran 414 tests and found four obsolete expectations in the text, +boolean, floating and string-numeric binding suites: each still required an +absent column to throw. The implemented families now correctly infer those +columns. Update only these expectations, retaining their unsupported-pair, +parameterized-type and designated-column rejection checks. Packaging checks +did not run after this failed gate; do not count them as passed. + +The corrected canonical client gate passed 414 tests plus two packaging checks. +The standalone client was then installed and frozen at SHA-256 +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. +The first server run stopped at compilation because the new test imported +`LineSenderServerException` from the wrong package. Only that test import was +corrected; the failed log is retained and no server execution was counted. + +Replay evidence is more specific than a generic assurance but not a new +inferred-Sender crash test: the existing, rerun +`QwpSchemaReplayNetworkTest.testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames` +persists manually encoded legacy and UNKNOWN-schema frames through +`CursorSendEngine`, reopens through public Sender, and compares both replayed +payloads byte-for-byte. The new inference tests separately prove exact emitted +UNKNOWN bytes and retention across a producer schema-generation replacement. +Existing public-Sender process-restart tests remain in the server gate; do not +describe these separate observations as one new inferred-Sender crash scenario. + +The first compiled server run passed all six new inference cases; the migrated +existing test failed only because its SQL used the reserved word `column` +unquoted in ORDER BY. Quoting that identifier preserved the expected schema +and data. The corrected focused gate passed all 14 cases with the frozen client. +Those runtime results now establish actual ACK adoption, source-native creation, +timestamp-only micro/nano/Instant/atNow behavior and write-time policy checks. + +One existing server diagnostic remains unchanged: disabled creation of a missing +table returns INTERNAL_ERROR (`failed to create table update details`) and then +uses the normal poison-frame terminal escalation. Create/add-column authorization +denial returns SECURITY_ERROR. The tests assert the current distinctions and no +table/column creation; this client increment does not reclassify server failures. + +### Final acceptance and remaining work + +The implemented families are BOOLEAN, LONG, FLOAT, DOUBLE, text as VARCHAR, +SYMBOL, UUID, BINARY and primitive/Instant timestamps. Known server targets +continue to select conversion; absent targets use existing buffer definitions. +No additional binding registry, producer/I/O state, protocol variant, opt-out +or conversion family was introduced. Relative to the saved 2.17 source, the +binding adds 88/removes 60 lines, the buffer adds a five-line package-private +name lookup, and Sender changes only the now-stale unbound-error comment. +Known-column writes do not use the added inferred-column lookup. This is a +source-level cost observation, not a throughput or allocation benchmark. + +Evidence is recorded in client `core/target/schema-inference-2.18/run-manifest.md` +and server `core/target/schema-inference.2.18-server/run-manifest.md`: + +- Client `logs/focused-2.log`: 140 tests passed. +- Client `logs/canonical-2.log`: 414 tests and two packaging checks passed. +- Client `logs/install-1.log`: standalone install succeeded; main JAR SHA-256 + `102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b` stayed + unchanged throughout all final server runs. +- Server `logs/focused-acceptance.log`: 14 tests passed. +- Server `logs/canonical.log`: 261 tests passed, preserving the prior 251-test + gate and adding six inference cases plus four existing auto-create checks. +- Server `logs/broad.log`: the unchanged 333-test selection has 10 failures, + 44 errors and zero skips. Root independently compared exact method names: + only `QwpSenderE2ETest.testAutoCreateBinaryColumn` and `testUuid` disappeared + from the prior 56; no failing method was added. +- Server `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 16 integration/recovery cases passed + each seed pair. These are repeated executions, not 48 different scenarios. + +The final broad failures classify as 38 missing conversions, five approved +schema/legacy expectation differences, four old direct-buffer/frame fixtures +and seven diagnostic expectations. Two former inference/order cases now reach +their later assertions: binary argument validation expects the old message +capitalization, and the column-mismatch test expects old text after a flush +that does not await ACK. The latter now sees the required MISSING-to-KNOWN +refresh and `SCHEMA_CHANGED`. Do not change production semantics to satisfy +these stale messages. Classifications describe this selected suite, not an +exhaustive count of all release gaps. + +Root verified the canonical server fork's actual startup temp argument and +open database descriptors under the project-local directory (PID 4123391), +not only Surefire XML. SOL independently reviewed the final production path, +wire tests and failure transitions. Source/test review mistakes were corrected +against actual protocol bytes and runtime results before acceptance. Existing +dirty work was preserved; no commit or push was made. + +Reflection: the existing buffer already had the necessary ownership and state. +The useful additional behavior is confined to selecting an inferred type and +reporting a genuine conflict through the existing refresh path. Read both the +public API lifecycle and the actual wire format before adding guards or tests. +Next, repair stale diagnostics while preserving independent legacy rejection +coverage, then continue bounded conversion families. Unsupported inputs, +old-distribution compatibility and the proposed lookup-timeout configuration +remain release work; this increment does not claim complete feature readiness. + +## D034 — Iteration 2.19: repair diagnostics without changing behavior + +Recommendation: fix the seven stale diagnostic expectations before adding +another conversion family. This is server-test-only work. The accepted 2.18 +client JAR stays frozen at SHA-256 +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +The selected tests now encounter schema-directed setters, so old generic error +text and asynchronous error callbacks no longer describe the operation being +tested. This is not a reason to change production behavior or accept either +local or server errors interchangeably. + +Source-backed decisions: + +- Assert `LineSenderSchemaException` and its reason for local invalid values or + unsupported conversions. Check useful table/column/type context without + pinning incidental capitalization. +- Preserve independent legacy BINARY-to-CHAR/STRING/SYMBOL/VARCHAR rejection + coverage. A nonrequesting WebSocket client sends an unflagged frame; parse + and assert its TYPE_BINARY definition and exact opaque bytes before sending. + Assert server SCHEMA_MISMATCH and zero rows, then separately test public + Sender's local rejection and partial-row recovery. `QwpWalAppender` explicitly + rejects TYPE_BINARY in all four target branches. +- Await the first row's ACK in the inferred-LONG mismatch test. In + `CursorWebSocketSendLoop`, schema feedback is applied before the acknowledged + sequence advances. This establishes the known LONG target before rejecting + fractional 3.14; a valid 3.0 then checks recovery. A sleep or an assertion + accepting multiple reasons would hide the actual lifecycle contract. +- Keep the UUID test's exact completed-A/failed-B/successful-C SQL result. + Exercise recovery without manual `cancelRow`; the binary cases additionally + continue without table reselection. + +SOL owns the two narrowly edited server test files and serialized validation; +a separate SOL reviewer checks compatibility, rollback and test independence. +Root checked the setter/flush/ACK ordering and the server rejection branches. +No production code, general test framework, private-state hook or opt-out is +needed. No client rebuild/install is planned, and old server distributions are +not covered by these current-server legacy-frame tests. + +Evidence limitation: the requested pre-edit copies of the two server files +were not captured by the authors. Do not reconstruct them and call them +snapshots. Review uses the observed original methods, existing worktree diff +and D033's recorded test evidence; the final source hashes and exact failure +method comparison will be recorded separately. Existing unrelated dirty work +must remain intact. + +### Final acceptance + +The iteration is accepted for the seven diagnostic tests. Both authors and an +independent SOL reviewer checked the source; root also checked exact wire, +setter rollback and ACK ordering. A stale comment claiming BINARY still passed +unchecked into VARCHAR was removed. Root requested continuation without table +reselection and an empty-flush assertion so the test observes recovery rather +than supplying cleanup itself. + +Server evidence lives in `core/target/schema-diagnostics.2.19-server/`, including +the exact commands in `run-manifest.md` and preserved logs: + +- `logs/focused.log`: seven affected tests passed. +- `logs/canonical.log`: the previous 261-test green selection plus seven + diagnostic tests passed, 268 total; no skips. +- `logs/broad.log`: the unchanged 333-test selection ran with seven failures, + 40 errors and no skips. Root independently compared method names against + D033: the six owned `QwpSenderE2ETest` diagnostics and + `QwpWebSocketSenderReceiverTest.testColumnTypeMismatchThrowsClientSide` are + exactly the seven removals; no method was added to the failing set. +- `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 23 selected diagnostic/integration/ + recovery tests passed each seed pair. These are repeated executions, not + 69 distinct scenarios. + +The remaining 47 classify as 38 missing-conversion cases, five approved +schema/legacy expectation differences and four obsolete direct-buffer/frame +fixtures. This is the selected regression suite, not an exhaustive release +backlog. Source-null text conversion decisions, unsupported input families, +actual older-distribution compatibility and the proposed lookup-timeout +configuration remain separate work. + +Final server source SHA-256 values: + +- `QwpSenderE2ETest.java`: + `4246989e6899c27035a37a3b7a586028a6d001f0704f6151880d3a6c8d132219`. +- `QwpWebSocketSenderReceiverTest.java`: + `f167130d6770f3c1c2b8961ab29cd2a47cc3fb1253d34f6eb4150230d3cc5990`. + +Before/after artifact and test-source hashes match. Root independently verified +the canonical fork's startup `java.io.tmpdir` and open database files under the +project-local directory (PID 4145918). No client rebuild/install or client test +rerun was needed; D033's 414 client tests plus two packaging checks remain +previous-iteration evidence, not new 2.19 executions. Diff checks pass; existing +dirty work was preserved, and no commit or push was made. + +### Next-step reflection: source nulls before text output + +Read-only planning identified a contract decision before LONG-to-STRING/VARCHAR/ +SYMBOL. The server's `putFixedToStringColumn`, `putFixedToVarcharColumn` and +`putFixedToSymbolColumn` check `cursor.isNull()` before decimal formatting. +`QwpFixedWidthColumnCursor.advanceRow` recognizes LONG's sentinel only when +there is no null bitmap; a supplied `Long.MIN_VALUE` marked present by a bitmap +therefore becomes decimal text instead. D013's approved client exception is +limited to numeric targets and does not silently authorize text targets. +Correction from D035's first runtime test: the formatter emits literal text +`"null"`, not decimal minimum. The bitmap-dependent NULL-versus-text contrast +is real; the initially predicted text was wrong. + +An initial SOL suggestion that UUID-to-text avoided this decision was rejected +on source review and retracted. Both public UUID paths always append the two +limbs; neither treats `(Long.MIN_VALUE, Long.MIN_VALUE)` as a no-op. The same +server cursor checks the UUID sentinel only without a bitmap, and both text +appenders otherwise format it as a UUID. It is not a decision-free alternative. + +Recommended next step: characterize these supplied-sentinel text cases through +the public and legacy-wire paths, then explicitly agree that source nulls remain +null before formatting in schema mode. That would be simpler than reproducing +batch-dependent behavior, but it is a proposed compatibility exception, not an +implemented or newly approved contract. No new converter is added in 2.19. + +## D035 — Iteration 2.20: characterize source nulls before text conversion + +The user requested proceeding from D034's next step. Scope: establish runtime +evidence for the five disputed pairs, then recommend an explicit contract. +This is characterization, not conversion activation. No production code, +public API, negotiated mode or server acceptance rule is changed. The 2.18 +client artifact remains frozen at SHA-256 +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +Source review covers public Sender, the non-owning binding, native column +appends, server dispatch, fixed-width cursor null handling and the actual +formatters. LONG-to-STRING/VARCHAR/SYMBOL and UUID-to-STRING/VARCHAR are accepted +legacy QWP conversions. UUID-to-SYMBOL is not included or newly authorized. +The supplied sentinels are effective writes, not no-ops: LONG minimum is +appended as eight bytes, and the UUID pair as two eight-byte limbs. Only the +server cursor's interpretation depends on the presence of a bitmap. + +SOL owns one new server test class and a separate SOL agent owns serialized +validation; an independent SOL reviewer checks the contract and assertions. +The new file's pre-edit state is explicitly absent. Existing production and +fixture files stay unchanged; no opt-out, private-state test hook or general +conversion/test framework is needed. + +Implemented and validated test matrix: + +- Legacy unflagged LONG frames to STRING, VARCHAR and SYMBOL: ordinary signed + values and range boundaries, including MIN+1/MAX, plus supplied MIN without + a bitmap; contrast a frame containing supplied MIN and an omitted companion + row. Assert source type, original bytes, exact bitmap bits, SQL text and + `is null`, not merely the cursor's interpretation. +- Legacy unflagged UUID frames to STRING and VARCHAR: an asymmetric ordinary + UUID, either limb alone equal to MIN and both limbs equal to MIN; contrast + the all-MIN pair without and with an omitted companion row. Assert raw + limb order and canonical text explicitly rather than using the production + formatter to compute expected values. +- Public schema-aware Sender for all five pairs: ordinary and sentinel native + inputs remain unsupported in the current build. Assert typed local rejection, + completed row A, rollback of partial B and successful text row C without + manual cancellation or table reselection. This does not claim that the + proposed converters or proposed null normalization are implemented. + +Recommendation, now runtime-backed but pending explicit contract agreement: +when these five conversions are implemented, normalize the supplied source +sentinels to target SQL NULL before formatting, independently of surrounding +rows. Keep effective-write and duplicate rules; do not reinterpret them as +setter no-ops. All other LONG values use signed decimal text, and all other +UUID limb pairs use canonical lowercase UUID text. This is a deliberate +exception to bitmap-present legacy results, not a claim of exact parity. +Do not extend it to FLOAT/DOUBLE, IPv4, LONG256, dates, timestamps or other +targets by analogy. Public UUID setter documentation currently does not state +this sentinel rule; implementation must document the approved scope. + +### First runtime findings and oracle corrections + +The first focused run selected all three tests: UUID characterization passed, +LONG characterization failed on its first bitmap-present STRING target, and +the public recovery case reached an invalid query-assertion configuration. +The log is preserved at server +`core/target/schema-text-source-null.2.20-server/logs/focused.log`. + +The LONG result was literal text `"null"` with `v is null = false`, not decimal +minimum. The supplied eight-byte MIN and row-0-present bitmap were exactly as +asserted. Root and SOL had traced the appender to `Numbers.append` but had not +read that overload's own sentinel handling. `Numbers.append(sink, long)` calls +the overload with `checkNaN=true`; its MIN branch writes `"null"`. The variant +with `checkNaN=false` would print decimal minimum, but is not called here. +Correct the expected literal and current design table; do not change production +or confuse the string `"null"` with SQL NULL. D034's predicted decimal text is +explicitly corrected above. UUID's formatter has no corresponding special +case and its source/byte/SQL expectations passed. + +The second failure is fixture misuse, not ingestion: `returnsOnce()` rejects +`.timestamp()` configuration. Remove that unnecessary setting while retaining +the exact SQL row values, timestamp strings and ordering. Adding `expectSize()` +would not make that configuration supported. This illustrates why checking +the test helper's actual contract matters as much as checking the formatter. +All subsequent validation must use a newly frozen source hash and separate +logs; no broad or repeat gate ran against the initial failing draft. + +### Final acceptance and next decision + +The corrected characterization is accepted. Source and adversarial SOL review +found no remaining issue after the two corrections; root independently checked +the actual formatter overload, query helper, frozen source and run results. +Automatic flush triggers are pinned using the existing configuration so row A +remains buffered across B's error. Legacy sockets are owned directly by +try-with-resources, including handshake failure; no extra ownership flag is +needed in a helper. + +Final new test SHA-256: +`47f9bf1ac588ed7c2b6ca74235a330a6f8c6e70adf8a90a6ba05ab9fd05bf816` +for server `QwpSchemaTextSourceNullE2ETest.java`. +Server evidence is under `core/target/schema-text-source-null.2.20-server/`: + +- `logs/focused-acceptance.log`: three tests passed, covering all five pairs + through loops; zero failures, errors or skips. +- `logs/canonical.log`: 271 tests passed, the previous 268 plus the three new + characterization tests; zero failures, errors or skips. +- `logs/broad.log`: the original 333-test selection has seven failures, + 40 errors and no skips. Root independently compared exact method names: + the same 47 as D034, with neither additions nor removals. The three new tests + are deliberately outside this unchanged comparison selection. +- `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 26 selected characterization, + diagnostic, integration and recovery tests passed at each seed pair. + These are repetitions, not 78 distinct scenarios. + +`run-manifest.md` records exact commands, the preserved initial red run and +initial source hash, and corrected before/after source/artifact hashes. +Root independently verified canonical fork PID 4168481's startup temp argument +and open database/WAL files beneath the project-local directory. The new test +source stayed frozen through all accepted runs; the two 2.19 fixture hashes and +workspace/installed client artifact remain unchanged. Client production code +was not changed, rebuilt or retested. Existing dirty work was preserved; diff +and whitespace checks pass. No commit or push was made. + +The evidence establishes the current server's legacy-frame behavior and the +current schema Sender's rejection/rollback. It is not proof of an older server +distribution, implemented normalization, or improved conversion coverage. +Keep the two independent legacy characterization tests when implementing the +converters; migrate only the corresponding current-public-rejection cases to +the approved behavior. Those rejections are a baseline, not a permanent goal. + +The next product decision is explicit: should these reserved LONG/UUID values +always become SQL NULL for the five text/SYMBOL target pairs, instead of +sometimes becoming literal text? Recommendation: yes, with legacy mode and +persisted legacy bytes unchanged. Document this compatibility exception before +implementing it; do not add batch-sensitive or send-time transformation. + +## D036 — Plan the remaining 47 repairs — 2026-09-11 + +The user requested an execution plan for all remaining failures, superseding +the tentative next LONG-to-text implementation. The separate +[repair plan](schema-aware-sender-remaining-failures-plan.md) is the execution +ledger; the main design remains the product contract. No production or test +source was edited, and no Maven/test execution was performed for this plan. + +### Evidence and sequencing + +Root and SOL audited the exact D035 broad-failure list against complete affected +tests, public setters/bindings, server dispatch, and concrete parser/formatter +implementations. The baseline remains 333 tests with seven failures and 40 +errors, not 47 distinct production defects. A mechanical comparison of the +plan's ledger with the preserved list found 47 entries, 47 unique methods, +no missing methods and no extras. + +Recommended order, divided into 16 independently reviewable increments: + +1. Nine test repairs: five approved schema/legacy expectation differences and + four fixtures that mutate Sender-owned buffers. Restore independent raw + legacy frames; do not weaken production behavior or add a schema opt-out. +2. Eight text conversions: LONG, UUID and FLOAT/DOUBLE input families. +3. Four ordinary timestamp input/output conversion cases. +4. Four CHAR, LONG256 and GEOHASH target cases. +5. Twenty-two decimal cases: native decimal first, then LONG, STRING, + FLOAT/DOUBLE and finally decimal text output. + +Every increment must retain a working public Sender real-server test, exact +wire assertions, independent legacy expectations, rollback/recovery coverage, +serialized frozen-artifact regression runs, seeded repeats and independent SOL +review. Count reductions are checkpoints, not permission to delete cases or +weaken assertions. A first fixed setter can expose a later stale assertion; +classify it against the approved contract before changing it. + +The plan uses the frozen 2.20 JAR and run manifest as its starting evidence. +The previous 271-test green gate and three 26-test repeats are historical +accepted results, not new planning-turn runs. Future increments must grow that +gate and compare exact names in the unchanged 333-test diagnostic. + +### Simplicity decisions and review corrections + +- Schema metadata already carries the complete packed server column type. + `QwpSchemaControl` writes it with an empty extension block; the client retains + it. GEOHASH bits and decimal precision/scale can use this existing metadata. + Do not invent new parameter fields, a second schema cache, or a new registry, + and do not relax rejection of unknown extension blocks. +- Decimal conversion belongs at setter time. One review suggestion to preserve + arbitrary input scale in schema-bound wire data and leave rescaling to the + server was challenged and withdrawn: it contradicts early conversion + validation. Convert non-null coefficients to target scale/precision and + compatible wire storage, then reuse existing append storage. Decimal-to-text + is different: it preserves source scale because that scale is visible text. + Cover emitted all-null columns separately from null/no-op setters; do not + infer decimal null behavior from fixed-width cursors. +- CHAR tests must distinguish malformed raw UTF-8 from malformed public Java + text. The public UTF-16 input is replacement-encoded before the server sees + it; a raw-byte parser result is not automatically the public API result. +- FLOAT and `Instant` overload coverage goes beyond the exact failing method + names but belongs with its corresponding input family. These extra tests + are not additional entries in the 47-method count. +- LONG/UUID text-null normalization remains an explicit contract prerequisite + for implementation; this plan does not silently extend it to floating-point, + timestamp or decimal inputs. Keep the independent 2.20 legacy characterization. + +The final independent SOL plan review found no remaining substantive scope, +dependency or ledger issue after those corrections. Root checked the ledger +and the relevant packed-metadata and decimal append/wire paths. Reviews inform +the plan; they do not substitute for runtime acceptance of its future changes. + +### Scope boundary + +All proposed functional changes are QWP client encoding/binding changes. SQL +only creates test tables and checks stored values; no SQL parser, planner or +execution-engine change is proposed. No send-time transformation, raw-type +fallback, reflection or new test-only production API is authorized by this plan. + +Zero remaining baseline failures will not establish full feature readiness. +Unrepresented conversion families, unsafe legacy-path compatibility decisions, +an actual older-server distribution, lookup-timeout configuration and broader +lifecycle/performance evidence remain visible release work in the main design. +No commit or push was made. + +## D037 — Iteration 2.21: separate schema expectations from legacy frames + +Status: accepted. The user approved proceeding with the repair +plan. This increment implements phase 1a only: five test repairs, plus one +public Sender counterpart to preserve integration coverage. No conversion or +other production behavior is changed. + +### Scope and source-backed decisions + +- The three Boolean conversion tests remain public Sender schema tests. They + keep every supplied true/false value and omitted row, but now assert the + target-native missing values and explicit nullness: BYTE/SHORT zero, nullable + numeric and text targets null. Independent raw BOOLEAN comparisons remain + in `QwpSchemaBooleanE2ETest`. One review initially recommended moving all + five tests to raw frames; root rejected that recommendation for the Boolean + cases, and the reviewer corrected it against the plan. +- The two floating legacy diagnostics now use nonrequesting WebSocket clients, + standalone native FLOAT/DOUBLE buffers and unflagged frames. They prove + no-bitmap NaN acceptance versus bitmap-present NaN rejection for both source + types and all four integer targets. Both source widths retain the accepted + predecessor, legacy clamp at positive 2^63, and rejected successor. Tests + assert raw bits, column definitions, bitmap bits, row counts/exhaustion, + precise server NACK context, stored values and empty rejected tables. +- Existing schema floating tests exercise bindings, not the public Sender + boundary path. Converting the two diagnostics solely to raw frames would + remove that integration coverage. The added + `testSchemaFloatingNaNAndPositiveTwoToThe63Rules` checks public FLOAT NaN in + a separately flushed block, DOUBLE NaN with an omitted companion row, and + local `INVALID_VALUE` for both positive-2^63 overloads. Completed row A and + valid row C survive two partial-row failures without manual cancellation or + table reselection. Existing auto-flush controls keep A buffered during errors. +- Root corrected a draft cursor assertion: no-bitmap supplied NaN is a cursor + null even though its raw value bytes exist; bitmap-present supplied NaN is + not. Do not replace the byte assertion with the cursor's interpretation. + Independent review also required the auxiliary label column's exact name + and VARCHAR wire type, not only its text bytes. Both fixes preceded the + first test run. + +The old NaN-only Sender error helper is removed because these cases now assert +server wire rejection directly. Shared Sender terminal-policy helpers remain +unchanged. No schema opt-out, shared test framework, reflection, new TestOnly +API or send-time conversion was added. SQL changes are only test queries that +check values and nullness; no SQL implementation was touched. + +### Frozen evidence and validation + +Server evidence directory: `core/target/schema-expectation-repair.2.21-server/`. +`baseline/` preserves both pre-edit test files; `run-manifest.md` records exact +commands, hashes and outcomes. The unchanged client JAR is +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. +Frozen changed test hashes: + +- `QwpSenderE2ETest.java`: + `9c48a484369d588114c291c8f6b8a2e939380ebb04c314523e00649b8ed9cab8`. +- `QwpWebSocketTypeConversionE2ETest.java`: + `253dcc2a8149a05fa57913868a19051f571d5b365d81ec431bbe08161da1a134`. + +Both independent SOL reviewers accepted the final source. Root independently +checked the diffs and current client/server behavior. The prior frozen 2.20 +broad log supplies the before-fix failures; no new red baseline run is claimed. + +- Focused: 25 tests passed, zero failures/errors/skips. +- Expanded green gate: 277 tests passed, zero failures/errors/skips. +- Original 333-test comparison: three failures and 39 errors, zero skips. + The exact 42-method set is the previous 47 minus the five phase-1a repairs, + with no new failing method. No case was removed or renamed. +- Three seeded repeats: 32 tests passed at each seed pair (`1/2`, `42/43`, + `1234567/7654321`), zero failures/errors/skips. These are repeats, not 96 + distinct tests. The corresponding Surefire properties confirm each seed pair. + +Root verified live canonical fork PID 5929's startup temp path and database +descriptors under the project-local directory. Surefire XML confirms the same +temp path and standalone installed client JAR. No client build or client test +run is claimed for this test-only increment. Both changed test sources and +the workspace/installed client JAR retained their frozen hashes through all +runs. Root and an independent SOL reviewer separately verified the exact +five-method removal with no additions. Diff and whitespace checks pass. +Existing unrelated work was preserved; no commit or push was made. + +Next: phase 1b, the four direct-buffer legacy fixtures. The 38 missing-conversion +cases are unchanged. This test-only repair does not implement additional +conversions, approve new null rules, prove an older server distribution, or +make the feature release-ready. + +## D038 — Iteration 2.22: isolate four legacy wire fixtures + +Status: accepted. The user requested the next increment. Scope +is phase 1b only: the two GEOHASH-to-text and two named LONG-to-designated- +timestamp tests in `QwpWebSocketTypeConversionE2ETest`. Existing 2.21 repairs +and all original stored-value expectations are preserved. + +### Decisions and source validation + +The old fixtures obtained a Sender-owned buffer, appended directly to it and +then called public row completion. That bypassed the producer's binding and +column tracking while row completion could negotiate/install schema state. +Do not accommodate that test-only mixture in production. The repaired tests +own standalone native buffers and send ordinary unflagged frames through a +nonrequesting WebSocket connection. Public Sender conversion behavior is tested +separately; no opt-out or new production testing API is needed. + +Root and SOL checked the concrete server paths, not just type names: + +- Positive GEOHASH precision 5 selects binary text formatting, not base32. + Preserve `[22, null, 31, null, 10]` and `[1, null, 0, null, 21]`, including + the non-null zero and leading zero bits. Explicit original timestamps keep + row order deterministic. The GEOHASH column is emitted last and checked as + exact bytes: bitmap marker 1, bitmap `0x0a`, precision 5, three compact values. + Cursor checks separately verify precision, row nullness/values and exhaustion. +- A named designated `TYPE_LONG` column is not timestamp-unit conversion input. + `QwpWalAppender` applies neither micro-to-nano nor nano-to-micro scaling to + this wire type; its designated fixed-width branch copies literal values. + Tests pin both column definitions, absent bitmaps, 8-byte widths, two values, + raw value/timestamp bytes and rowwise values, then exact micro/nano SQL output. +- The existing public Sender designated tests already cover both precisions, + primitive/Instant input, partial-row rollback and cold `atNow()`. They remain + in the focused/regression/repeat gates. Unlike phase 1a, no missing public + counterpart required a new test method. Native public GEOHASH conversion + remains unimplemented and must not be credited to these legacy fixtures. + +Root found one review correction before execution: the draft ACK parser passed +`true` for `schemaNegotiated` despite using a nonrequesting connection. The +final default overload rejects unexpected schema feedback. Exact table names +and native LONG byte assertions were also added. Both independent SOL reviewers +accepted the refrozen source after these corrections. Helpers remain local to +the test class; class documentation now distinguishes public conversion tests +from raw legacy server tests. No reflection, shared framework, or production +change was introduced. + +### Frozen evidence and validation + +Server evidence directory: `core/target/schema-legacy-fixture-repair.2.22-server/`. +The pre-edit source is preserved in `baseline/`; `run-manifest.md` records exact +commands and results. The changed test source moved from +`253dcc2a8149a05fa57913868a19051f571d5b365d81ec431bbe08161da1a134` to +`726112e1bc4c88befd0f40df088ce5409485f59b170d707829a214debafa260c`. +The prior Sender test remains `9c48a484369d588114c291c8f6b8a2e939380ebb04c314523e00649b8ed9cab8`. +Workspace and installed client JAR remain +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +- Focused: 37 tests passed, zero failures/errors/skips. +- Expanded green gate: 281 tests passed, zero failures/errors/skips. +- Original 333-test comparison: zero failures, 38 errors, zero skips. Exactly + the four phase-1b methods disappeared from the prior 42; no new failure. +- Three seeded repeats: 36 tests passed at each pair (`1/2`, `42/43`, + `1234567/7654321`), zero failures/errors/skips. These are repetitions, not + 108 distinct scenarios. Each run's Surefire properties confirm the seed pair. + +No client build/test run or new red baseline is claimed. The accepted 2.21 +broad log supplies the original four failures against the preserved source. +Root observed canonical fork PID 22089's startup temp argument and checked its +completed Surefire path/artifact properties. Broad fork PID 24170's startup +argument and live database descriptors independently confirm project-local +storage. No shared `/tmp` database workaround or overlapping Maven run was used. + +The changed test, previous Sender test and workspace/installed client JAR stayed +at their frozen hashes through all runs. Root and an independent SOL reviewer +separately confirmed the exact four-name removal. No method was renamed or +deleted. Diff/whitespace checks pass; unrelated changes were preserved. No +commit or push was made. + +Phase 1 is complete: all nine expectation/fixture repairs are accepted. The +remaining 38 baseline cases require missing conversions; this does not finish +the complete feature-release checklist. Next is LONG-to-text, with the proposed +source-null rule recorded as an explicit contract before implementation. + +## D039 — Iteration 2.23: approve LONG-to-text source nulls + +Status: accepted on 2026-09-11; contract, implementation and validation complete +for phase 2a only. +The user's “makes sense” approves the preceding LONG-only recommendation: +in schema mode, `longColumn` writing to STRING, VARCHAR or SYMBOL converts +`Long.MIN_VALUE` to target SQL NULL, independently of other rows. This is an +effective first write, so subsequent duplicates remain ignored. Other LONG +values use canonical signed decimal text. Legacy encoding, conversions and +replay remain unchanged. UUID and floating-point text-null rules are not +approved by analogy. + +Two SOL source checks confirmed that existing VARCHAR and dictionary-backed +SYMBOL append paths suffice. Both copy mutable formatter input before returning; +reuse of a formatter sink does not require changing dictionary ownership. +Missing-table/column inference must remain native LONG. Resolve target and +duplicate rules before handling the sentinel; unsupported target pairs must +remain unsupported even for a sentinel input. + +Scope is phase 2a only: three baseline failures, with exact target wire, +public Sender, real-server stored-value and recovery tests. Keep the independent +2.20 legacy sentinel/bitmap characterization and all unsupported UUID checks. +No SQL, server production, send-time transformation or new conversion framework +is proposed. Test preparation was delegated to SOL. Production edits awaited the +pre-edit checks at the decision point; the completed work is recorded below. + +### Implementation and review + +The production change is confined to `QwpSchemaBinding`: permit the three text +targets in `longColumn`, retain target/duplicate validation before the existing +MIN null branch, format other values through `Numbers.append(..., false)`, then +use existing VARCHAR or SYMBOL append paths. One lazily allocated 20-character +sink belongs to the non-thread-safe binding. Numeric writes never format or +allocate that sink; new distinct symbols still require the existing owned +dictionary entry. No new cache, registry, writer, protocol or connection state. + +The shared 18-vector corpus has literal expected UTF-8 and SQL values. Tests +cover MIN+1/MAX, signs/zero, MIN-first followed by an otherwise-invalid UUID +duplicate, omission bitmaps, reset, native LONG inference, and local/global +SYMBOL sequences `42 -> 43 -> 42` to detect retained mutable formatter input. +Public Sender socket tests pin the actual described identity and target bytes, +cancel partial B while preserving A/C, and retain separate SYMBOL/VARCHAR +encoding generations after an explicit setter-triggered refresh. The new +real-server test checks all pairs and SQL nullness. Its SF case seeds a local- +dictionary frame into the real log and replays it through a new Sender; it is +not described as public-Sender crash persistence. Existing public crash/global- +dictionary and two-generation recovery tests remain in the server gates. + +Independent SOL source and adversarial reviews accepted this bounded change. +Root preserved these findings rather than adding production workarounds: + +- A cancelled row can leave an unused global symbol entry. The dictionary is + append-only; row rollback does not rewind it. A draft socket oracle incorrectly + expected B's `7` entry to disappear. The corrected oracle requires `[7, MAX]`, + MAX at ID 1, and only A/C rows. No dictionary-rollback mechanism was added. +- The first broad comparison found another stale LONG-to-STRING rejection in + `QwpSchemaRowBufferE2ETest.testNullOmissionAndUnsupportedUsedColumns`. Change + only its unused target declaration from STRING to BINARY: LONG-to-BINARY is + genuinely unsupported, as confirmed by `QwpWalAppender`'s BINARY branch. + The method name, cancellation, omission checks and exact SQL expectations + remain unchanged. Add this method to the green/repeat gates. Do not substitute + UUID-to-STRING, which the next planned increment intends to support. +- Compile-only fixture mistakes (client Unsafe API, response constant types, + server helper visibility/checked exceptions) and a malformed MISSING-schema + response are retained in development logs, not counted as behavioral red + evidence. Source review did not replace compilation. Valid red baselines + then showed five client conversion errors with one native-inference pass, + and all three new server tests failing on the missing conversion. + +The old mixed text-rejection test now retains only the four UUID ordinary/null- +sentinel cases; its independent legacy LONG/UUID byte/bitmap tests are unchanged. +The original three LONG-to-text methods retain their existing SQL expectations. + +### Focused performance evidence + +The hardware-first skill caused a pre-edit baseline check, not an optimization. +The validated study in `core/target/schema-long-text.2.23-client/perf/` is limited +to the existing warmed LONG-to-LONG binding/readback case. Six fixed-work +baseline runs preceded the production edit; twelve interleaved control/candidate +runs followed. Every run completes 500 million rows with the same checksum. +Basic and pipeline-counter median elapsed differences are +0.233% and +0.155%, +below observed variation; instructions/row are effectively unchanged. No +repeatable 5% regression was detected for this case. This is not proof of +sub-percent equality, new text-conversion speed, or end-to-end Sender throughput. +CPU affinity was fixed, but shared-host frequency/boost/thermals were not +controlled. Independent SOL review verified counts, identities and this scope. + +Final server comparison and seeded acceptance results follow. The first broad +run is retained: 333 tests, one stale fixture +failure and 35 missing-conversion errors; it is not the accepted final baseline. + +### Frozen evidence and acceptance + +Client evidence: `core/target/schema-long-text.2.23-client/run-manifest.md`. +Server evidence: `core/target/schema-long-text.2.23-server/run-manifest.md`. +Both include exact commands and distinguish failed development runs from final +gates. No client install or benchmark overlapped server validation. Every test +fork received a pre-created project-local temporary directory through startup +`argLine`; root observed canonical PID 66407's command line and broad PID 68499's +command line/open database descriptors. The canonical descriptor inspection +missed the already-exited process and is not claimed as successful. Saved +Surefire reports confirm temp paths, standalone installed client dependency and +all three seed pairs. + +- Client focused: 6 tests passed. Expanded client gate: 420 tests plus two + packaging checks passed; zero failures/errors/skips. +- Server final focused: 16 tests passed. Expanded green gate: 288 tests passed; + zero failures/errors/skips. Earlier 15/287 runs preceded the additional stale + fixture repair and are retained separately. +- Original unchanged selection: 333 tests, zero failures, 35 errors, zero skips. + Exactly `testLongToStringColumn`, `testLongToSymbolColumn` and + `testLongToVarcharColumn` disappeared from 2.22's 38-name failure set; no added + failure. Root and an independent SOL reviewer compared the sets separately. +- Final seeded repeats: 43 tests passed at each of `1/2`, `42/43` and + `1234567/7654321`, zero failures/errors/skips. These are repetitions, not 129 + distinct scenarios. An earlier 42-test repeat is retained, not substituted + for the expanded final selection. + +Client binding moved from +`cf1415484f32067b9135e235d1a002607d1a99e5dd195c8c7d558aaae111aaca` +to `e24350a9522ceecb64267038410c914dd6b305299c146d5a9c9974c61cbac600`. +Workspace/installed main JAR moved from +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b` +to `0d2d66ce80e7fb20fd3e7468b70ad2d49888b60cb0108694e30f0d713a8ecb49`. +Client/server corpus SHA-256 is +`0db1278342169f2291fa84027c7179a2097549fce1e1eec4aac60b1dab8988c7`. +Final source hashes, including the three changed server test classes, are in +the manifests. Buffer, Sender, original type-conversion/Sender fixtures and +server production remain unchanged through validation. Diff/whitespace checks +pass; unrelated working-tree changes were preserved. No commit or push. + +Phase 2a is complete. Of the original 47 cases, 35 remain; this is not full +feature-release acceptance. Next is UUID-to-text, but its proposed both-limbs- +MIN null rule still needs a separate decision. Do not infer approval for UUID, +floating, timestamp or decimal null behavior from this LONG-only increment. + +## D040 — Iteration 2.24: UUID-to-text with explicit source nulls + +Status: accepted on 2026-09-11 for phase 2b only. The user's "proceed" explicitly +approved the separately proposed UUID rule: for STRING/VARCHAR targets, both limbs equal to +`Long.MIN_VALUE` become SQL NULL; every other pair produces canonical lowercase +UUID text. A single minimum-valued limb remains a value. The null is an +effective first write, so later duplicates are ignored. This deliberately +differs from legacy bitmap-present formatting of the both-MIN pair. Preserve +legacy ingestion/replay and native UUID-to-UUID bytes. UUID-to-SYMBOL remains +unsupported; no floating, timestamp or decimal policy is inferred. + +Scope is phase 2b, two baseline cases. SOL was assigned client implementation/tests, +server E2Es and independent review. Tests must pin literal target bytes and SQL +nullness, limb order, sentinel boundaries, omission/duplicates, row rollback and +encoding snapshot retention. Keep the independent raw legacy characterization. +No server production, SQL implementation, send-time transformation or generic +conversion machinery is planned. + +The hardware-first skill required a pre-edit gate for the changed value-writing +path. A focused existing native UUID-to-UUID baseline and interleaved comparison +were scheduled to check regression risk, not new text conversion speed or +full Sender throughput. The accepted 2.23 binding and JAR were archived under +`core/target/schema-uuid-text.2.24-client/baseline/` before edits. Their hashes are +`e24350a9522ceecb64267038410c914dd6b305299c146d5a9c9974c61cbac600` +and `0d2d66ce80e7fb20fd3e7468b70ad2d49888b60cb0108694e30f0d713a8ecb49`. +At this pre-edit checkpoint, the last accepted baseline had 35 errors. + +Pre-edit source review found two LONG null-first tests whose UUID duplicate was +described as invalid. It becomes supported for STRING/VARCHAR in this increment. +Use genuinely unsupported BINARY-to-text/SYMBOL duplicates instead, retaining +the same null/first-write and rollback assertions. The SYMBOL-to-VARCHAR rebind +test still correctly uses UUID to trigger refresh: its pinned SYMBOL schema +rejects UUID, irrespective of the refreshed target. Keep it unchanged. + +Behavioral red evidence uses the frozen 2.23 production source/JAR. Client +focused selection ran 11 tests with three missing UUID-to-text errors and eight +passes. Server selection ran four tests: all three new conversion E2Es failed +with `UNSUPPORTED_FEATURE`; the narrowed UUID-to-SYMBOL recovery test passed. +Source review caught malformed draft schema-fixture parameter/designated fields +before compilation and corrected them to the real wire protocol. Neither these +draft errors nor the later reset-oracle correction are production findings. +Raw logs are preserved under each repo's `core/target/schema-uuid-text.2.24-*/`. + +### Implementation and client validation + +Only `QwpSchemaBinding` changes in production: target-directed UUID dispatch, +a dedicated lazy 36-character sink and five fixed-width lowercase hex groups. +The native UUID branch returns before text-null normalization. Existing target +validation and duplicate handling run before conversion, and `addString` copies +the scratch contents synchronously. No change to LONG formatting, shared +Numbers, Buffer, Sender or server production. Independent SOL source and +adversarial reviews accepted this scope. + +The expanded red client gate has 12 tests and four expected conversion errors. +The first candidate run exposed a test-oracle defect: all column definitions +precede all payloads, but the rollback reader expected them interleaved. Both +source reviewers confirmed the encoder contract; fix the reader only. The +corrected focused gate passes all 12 tests. The expanded client gate passes +426 tests plus two packaging checks, with zero failures/errors/skips. The new +public Sender test verifies typed UUID rows on both sides of a UUID-to-VARCHAR +schema change: the old block retains native limbs, the new block contains text. +Refresh is setter-triggered; no passive ACK-adoption claim is made. + +Current candidate binding SHA-256 is +`9d49b8476e844aae15227448d103372c18ebfe620cbb2bd4c49e42528d6743ff`. +After successful install, workspace and installed main JAR both hash to +`c06908f19e82b95625c1f3b5661a06a0fe9ced10637defde7d4f845d817c989d`. +The shared 16-vector corpus hashes to +`3278914c60022786037fb09380b717bfde23f327f750b8e33c938e14f0eaba03`; +root independently checked signed limbs against literal UUID text and UTF-8 hex. +Server acceptance and the interleaved performance comparison were pending at +that checkpoint; their completed results follow. + +### Regression checks and fixture corrections + +The pre-edit hardware study was corrected before acceptance: retired-operation +counts alone did not justify a limiting-resource claim, major faults were not a +CPU error metric, and a fixed footprint was not measured saturation. The final +study records actual pipeline fractions, normalized instruction/cycle costs, +fresh cgroup throttle/OOM deltas and qualified kernel-error visibility. Root +reran the validator and reviewed the evidence before authorizing production. +Six baseline comparisons and one separate systemic run preceded edits; the +systemic run started after client red validation finished. + +All twelve interleaved candidate/control measurements completed 500 million +native UUID rows with the same two-limb checksum. Basic instructions per row +changed by -0.186%; neither elapsed nor normalized instruction/cycle costs show +a repeatable 5% regression. The observed lower candidate elapsed/cycle costs +are not a speedup or mechanism claim: shared-host frequency/boost are uncontrolled. +This check covers native UUID-to-UUID only, not the new formatter, allocations, +public Sender or networking. Independent SOL review checked all twelve raw runs. +See `core/target/schema-uuid-text.2.24-client/perf/comparison.md`. + +The first server focused run passed 20 of 21 tests, including all shared vectors, +source-null/legacy checks and text replay. The new rollback test expected an +omitted LONG column to print blank; `CursorPrinter` and the actual output show +`null`. Change only the two expected cells, retaining the same projection, +rows and rollback checks. The failed source and log are archived. These small +fixture defects reinforce that source review does not replace execution; none +justifies a production workaround. + +### Frozen evidence and acceptance + +Client manifest: `core/target/schema-uuid-text.2.24-client/run-manifest.md`. +Server manifest: `core/target/schema-uuid-text.2.24-server/run-manifest.md`. +Both preserve exact commands, identities and unsuccessful development runs. +No Maven runs or client installs overlapped the fixed-work measurements; no +client rebuild overlapped server validation. Saved reports verify startup +project-local temp directories and the installed client dependency. Root also +observed canonical Java PID 121872 and its open database files under the expected +temp path. The focused live-process snapshots missed the short-lived fork and +are not claimed as successful inspections. + +- Client: final focused 12 passed; expanded 426 plus two packaging checks + passed; zero failures/errors/skips. Install succeeded. +- Server: final focused 21 passed; expanded green gate 293 passed; zero + failures/errors/skips. +- Original unchanged selection: 333 tests, zero failures, 33 errors, zero skips. + Only `testUuidToStringColumn` and `testUuidToVarcharColumn` disappeared from + 2.23's 35-name list; no added failure. Root and an independent SOL reviewer + separately checked both directions and the identical selection/fixture hashes. +- Repeats: 48 tests passed at each of `1/2`, `42/43`, and `1234567/7654321`. + Saved UUID test reports confirm each seed pair and temp path. These are + repetitions of 48 cases, not 144 distinct scenarios. + +Final binding and main-JAR hashes remain `9d49b847...6743ff` and +`c06908f1...17c989d` (full hashes above and in the manifests). Final client UUID +test SHA is `c23aa77b7edeff165bcef36ed60bce72425b45507d60e935227cbd53588ce1c6`; +server UUID test SHA is +`4371b0e3aab331755c4101fffde5384ec2966d01f29f8ecba650ea2273dc17ce`. +All frozen production/test/corpus identities remained unchanged through final +gates. Independent SOL cross-reviews accepted client production/tests and server +tests. Raw legacy LONG/UUID characterization is unchanged; the new SF test is +explicitly component-seeded replay, with public crash/two-generation recovery +retained in existing regression tests. Diff/whitespace checks pass. Unrelated +working-tree changes were preserved. No commit or push. + +Phase 2b is complete. Of the original 47 cases, 33 remain; the full conversion +inventory and release checklist are not complete. Next is phase 2c FLOAT/DOUBLE +text output, after separately characterizing and deciding its source-null rule. +Do not infer that policy from LONG or UUID approval. + +## D041 — Iteration 2.25: floating-point text output + +Status: accepted, 2026-09-11. The user requested that work continue one +increment at a time until the remaining conversions are fixed. Start with +phase 2c, then the timestamp, scalar-target and decimal increments in the plan. +The last accepted original selection still has 33 errors. Do not confuse that +finite test list with the wider source-audited conversion inventory or silently +implement the documented unsafe server boundaries. + +Keep production changes sequential, with SOL implementation and independent +review, real-server tests, exact target bytes, unchanged legacy comparisons +and frozen client artifacts. No SQL-engine changes or general conversion +framework. Reuse the hardware-first regression checks for affected existing +paths; no throughput claim is implied. + +The floating text-null rule remains a separate explicit decision. Source +inspection confirms the legacy cursor recognizes NaN as null only without a +bitmap; when a bitmap marks it present, the server formatter emits text `NaN`. +Root recommends deterministic target SQL NULL and has asked the user, without +treating the general continuation request as approval of that choice. SOL is +preparing raw real-server characterization while this decision is pending. + +A second load-bearing assumption failed source inspection: current server +`Numbers.append(double)` delegates to Ryu, while the client retains the older +double formatter. Both expose the same method/default scale. Differential +checks must establish output compatibility before reusing client formatting; +do not change shared legacy formatting to fix this schema-only feature. + +Accepted 2.24 binding, Numbers source and main JAR are archived in +`core/target/schema-floating-text.2.25-client/baseline/` before any production +edit. Binding/JAR hashes remain `9d49b847...6743ff` / `c06908f1...17c989d`. + +The user subsequently explicitly selected SQL NULL for NaN. This approves all +six FLOAT/DOUBLE-to-STRING/VARCHAR/SYMBOL pairs, with effective-write/duplicate +semantics, no legacy change and no inference for other source families. +The new raw real-server characterization passed: one test covering 12 +source/target/bitmap combinations, exact raw bits, null flags, stored text and +SQL nullness. Its source/report are archived under the server's +`core/target/schema-floating-text.2.25-server/` before schema-positive additions. + +The formatter differential found 1,543 mismatches across 150,004 supplied values +(275 DOUBLE, 1,268 widened FLOAT). Examples include `1e23` and the smallest +DOUBLE subnormal. Reusing old client Numbers is therefore not a simplification +that preserves the contract. Prefer an isolated source-faithful schema formatter; +leave global Numbers and legacy text formatting untouched. + +A parallel timestamp audit corrected a stale plan statement: the server first +converts nanoseconds to micros, but its text formatter emits milliseconds, not +microseconds. Root verified the call sites and UTC pattern and corrected phase +3c. This is a source-backed plan correction, not permission to change server +output. Timestamp sentinel/empty-text behavior still needs characterization. + +The pre-edit performance check completed six fixed-work native DOUBLE-to-DOUBLE +runs with identical raw-bit checksums. Root inspected the source attribution, +resource checks and normalized counters and independently ran the study validator. +This is a narrow regression baseline, not floating-to-text throughput evidence. +Production edits were cleared only after this check and the client red tests: +six tests ran, five failed with the expected missing-conversion error; the +native-inference/unsupported-target guard passed. The server red run then had +eight tests, seven expected missing-conversion errors and a passing independent +legacy characterization. Both red XML sets are archived in the 2.25 evidence +directories. Independent SOL source review requested one test strengthening: +compare the SF frame identity with its actual describe reply, not just its types. + +The candidate adds a schema-only formatter and an exact server Ryu port; only +package/class names and provenance differ in the latter. The binding adds lazy +scratch and the three text target arms. Its numeric branches, native inference, +target/duplicate ordering and public Sender code are unchanged. Main client +Numbers is byte-identical to the pre-edit source. Independent SOL review accepted +the frozen code and exact wire fixtures. + +Root reproduced the public-binding/encoder differential: seeds 225202601 and +225202602 generate one million DOUBLE and one million FLOAT bit patterns. +The 999,505 non-NaN DOUBLE and 996,216 non-NaN widened FLOAT results, plus 46 +structured cases, match the actual server formatter byte for byte. Random NaNs +are counted and skipped in this formatter comparison; the shared 48-vector +corpus and public tests separately assert their approved target-null behavior. +`core/target/schema-floating-text-audit/run.sh` records source identities and +the exact build/classpaths; `root-replay.log` records the independent replay. + +The six focused client tests and packaging checks passed. The first 432-test +client regression run exposed a stale numeric-only fixture that still rejected +NaN-to-STRING/SYMBOL. Those two negative cases now use unsupported TIMESTAMP and +BINARY targets, preserving their guards and rollback checks. No production fix +was needed. The rerun passes all 432 tests and two packaging checks. The installed +and archived candidate JAR SHA-256 is +`44fdc2bf48f83db01eed089f915ab451c8490ec8d0aea6fbd638c070142dd43b`. +Interleaved native-path performance and server acceptance gates remain pending. + +Read-only preparation for phase 3a confirms that LONG-to-ordinary timestamp needs +only target dispatch: both target units store the raw supplied count. For MIN, +the legacy no-bitmap null and bitmap-present raw value both store the same +SQL-null sentinel (`QwpWalAppender` ordinary TIMESTAMP branch and +`WalColumnarRowAppender.putIntegerToNumericColumn`). This requires no new +source-null policy exception. Keep designated-name rejection and native LONG +inference; do not route raw LONG through a timestamp unit converter. SOL may +prepare tests while 2.25 is validated, but no next-increment production edits +are permitted before this checkpoint is accepted. + +The first candidate's performance result is not accepted as unchanged: all three +paired basic runs add about 25 instructions per native DOUBLE row (+4.62%), +with roughly 2–4% elapsed cost across the event groups. Although narrowly under +the predeclared 5% rejection line, root requested source attribution before +acceptance. SOL confirmed a C2 inlining cliff: the numeric helper grew from 322 +to 420 bytecodes as its switch range expanded from 2–10 to 2–26. The baseline +inlines hot; the candidate is rejected as too large. This is a repeatable native +cost, not a text-throughput result or mere timing noise. + +Authorize only a small separation of the cold text branch, preserving the +original numeric switch and all validation/null/duplicate ordering. Predict +restored hot inlining and removal of the extra per-row instruction work. Retain +the original candidate and all A/B artifacts; require a new frozen artifact, +complete functional gates and interleaved performance comparison. Do not tune +JVM settings, suppress the evidence or lower the acceptance bar. + +The small split restores the numeric helper to 311 bytecodes; all numeric +arithmetic is unchanged. The first server candidate run reached 29 tests and +found one new expected-output mistake: an omitted UUID prints as an empty cell, +not numeric `null`. Correct only that fixture; the finite/NaN corpus, rebind, +SF and legacy checks passed. The earlier attempt failed test compilation in a +prepared phase-3a fixture (a string case label used as timestamp arithmetic). +That test-only mistake was corrected. Preparation of future increments must +not pollute the current test selection: archive future integration additions +outside the source tree until the preceding checkpoint is accepted. The live +integration class was restored to its exact 2.25 snapshot before the rerun. + +Final acceptance: the corrected helper inlines hot at 311 bytecodes. Twelve +new paired measurements have identical fixed-work checksums, and the first +candidate's extra 25 native DOUBLE instructions per row is gone. Basic cycles +show no regression; pipeline-group elapsed remains 0.73–2.99% higher, qualified +as a limited shared-host/group result rather than a speedup or universal +unchanged-performance claim. Root independently checked the raw comparisons, +inlining evidence and study validator. Both candidate histories are retained. + +The final workspace, installed and archived client JAR SHA-256 is +`1173fe3e8dd3fcd764c3513882e4461e608ff000e8d00d1dac6f91e8386f8b70`; +binding SHA is `bb4955a8c347f9b8c9fb6741f9a3b88e10cc8f471f5a8e78084dd5a31e278f1e`. +The final candidate passes 432 client tests plus two packaging checks; root +replayed the differential successfully after the split. Server gates pass +29 focused tests, 301 regressions and three 56-test repeats (seeds 1/2, 42/43, +1234567/7654321). Saved XML verifies startup tmp and seed properties. The exact +original 333-test selector has zero failures and 30 errors, removing only +DOUBLE-to-STRING/VARCHAR/SYMBOL from the previous 33-name list. No additions. +Root and independent SOL review verified the names, frozen source identities +and typed/null/lifecycle contracts. Server production, legacy Numbers, Buffer, +Sender and the original conversion-test source are unchanged in this increment. + +Evidence and commands: client and server +`core/target/schema-floating-text.2.25-{client,server}/run-manifest.md`, saved +reports, both performance comparisons and the separate differential runner. +No commit/push. Phase 2 of the remaining-failures plan is complete; 30 baseline +cases and the separately tracked wider conversion inventory remain. Phase 3a +has reviewed test preparation only, including two corrected fixture mistakes; +its production change must receive its own red tests, measurements and gates. + +## D042 — Iteration 2.26: LONG to ordinary timestamp + +Status: accepted, 2026-09-11. Continue phase 3a after the accepted 2.25 +checkpoint (30 baseline errors). This is a small extension of the existing +LONG setter, not timestamp parsing or formatting. A known ordinary micro/nano +timestamp column takes the raw LONG count in its own unit; there is no implicit +micro-to-nano multiplication. Supplied MIN remains an effective target-null +write. Source review confirms both legacy bitmap contexts store the same SQL +null, so this adds no source-null policy exception. Named designated timestamp +writes remain rejected, and missing schema still infers LONG. + +The implementation brief permits only the target allowlist and the two existing +target-native addLong switch arms. No timestamp driver, protocol field, new +buffer representation or conversion framework is needed. Root and SOL read +the producer path, target-column guards, wire mapping and ordinary timestamp +branches in QwpWalAppender/WalColumnarRowAppender before this brief. + +Prepared tests use a shared 12-vector corpus, exact target-unit bytes, null and +omission bitmaps, invalid duplicates, automatic A/B/C recovery and pinned native +LONG/timestamp generations. Real-server assertions inspect raw stored timestamps +and SQL nullness in both units; independent unflagged frames characterize MIN +with/without a bitmap. Preparation review caught and fixed two fixture mistakes: +MISSING replies carry no identity pair, and a textual corpus label is not an +epoch count. No production implementation has been authorized yet. The native +LONG baseline is being measured against the accepted 1173fe3e... JAR before +red tests and production changes; no concurrent Maven is permitted. + +Root read and independently validated the fresh native LONG baseline study. +Client red: five tests, four expected unsupported-conversion errors; the +native-inference/designated guard passed. Server red: four tests, three expected +unsupported-conversion errors; independent legacy MIN characterization passed. +The authorized production patch adds only two timestamp target checks and two +labels sharing LONG's raw addLong arm (binding SHA 1f1a0396...). + +The first focused client run found four wire-fixture failures. Both source +reviews had missed an existing timestamp encoding discriminator: the default +Gorilla-enabled stream writes raw-encoding byte 0 after the null bitmap even +for zero, one or two values. Root traced QwpColumnWriter and the existing +timestamp corpus test, then requested explicit assertions of that byte in all +new timestamp readers. Native LONG readers stay unchanged. Production did not +change. Record this as a review miss, not a converter bug: exact-wire fixture +reviews must trace the actual per-type writer and cursor, and never substitute +source review for the runtime red/green gate. The corrected fixtures are frozen +and under test; no generalized reader framework is introduced. + +The corrected client run passes all five focused tests plus two packaging +checks, and all 437 canonical tests plus two packaging checks. Final workspace, +installed and archived JAR SHA is +`35f3eaa6c0b663be59df12cc422346756ee120fa60d4ebde15e0d80d7f98cf58`. +Twelve paired native LONG measurements completed identical 500-million-row +work and checksum. Instructions are effectively unchanged, with no repeated +elapsed/cycle regression at the predeclared threshold, major faults or cgroup +throttle/OOM deltas. Root independently validated the final study/comparison. +This establishes no detected LONG-to-LONG regression, not timestamp throughput. + +Server focused (41) and canonical (305) gates pass. The unchanged original +333-test selector has zero failures and 29 errors; exact name comparison removes +only `testLongToTimestampColumn` from the previous 30. Both 1/2 and 42/43 seeded +68-test repeats pass. The final 1234567/7654321 repeat also passes all 68 tests. +Root checked selected XML seed/tmp properties, exact selector equality, +failure-name delta, installed/workspace/archive identities and both frozen +source manifests. Server appenders and original conversion-test source are +unchanged. Evidence and commands are in client/server +`core/target/schema-long-timestamp.2.26-{client,server}/run-manifest.md`. +No commit/push. Phase 3a is complete; 29 baseline errors and the wider inventory +remain, not a release-ready client. + +Final independent SOL acceptance directly verifies all raw server summaries, +the exact one-name delta, selected repeat XML and final artifact/source hashes. +No remaining finding for this slice. + +## D043 — STRING-to-timestamp: fixed parser, narrow timezone adapter + +Status: iteration 2.27 accepted, 2026-09-11. Preparation and implementation +decisions follow chronologically; final acceptance evidence is at the end. +Root and two SOL audits traced the actual QWP STRING path +through `WalColumnarRowAppender.putStringToTimestampColumn` and +`MicrosFormatUtils.parseTimestamp`, then its generated lexical rules, calendar +computation, English locale tokens and timezone rules. Source and public-parser +probes are under `core/target/schema-string-timestamp-preparation/`. + +Reject the initial false choice between a reduced ISO-only grammar and porting +the general compiler. Implement the eleven fixed formats in their real priority, +with exact relevant calendar helpers and a small zone adapter. The server accepts +hour 24 with modulo normalization even with nonzero minutes; fractional widths +are not a generic arbitrary-length ISO fraction. Preserve existing UTF-16-to-UTF-8 +replacement followed by the server's ASCII-byte parsing view. Do not parse Java +Unicode input more broadly than the bytes the server receives. + +The first zone proposal used JDK `ZoneRules.getOffset(Instant)` everywhere. +Root challenged its treatment of separately parsed years after microsecond +arithmetic wraps. The public-server probe disproves it: +`3509450-07-01 00:00:00.000Europe/Berlin` produces +`4929198542690304` micros on the server, while the naive JDK adapter gives +`4929202142690304`, a one-hour difference. Root independently replayed the probe. +The wrapped naive epoch is `4929205742690304`; server future rules still use +parsed year 3509450 rather than the wrapped instant's year. + +Use public immutable JDK rule metadata and preserve the server's split: fixed +and historical offsets through the exact cutoff boundary; recurring transitions +after it, calculated with the separately parsed year and source-faithful +arithmetic. Retain rule order and UTC/STANDARD/WALL adjustments. This requires +only the relevant transition calculation, not its general timezone engine, +Unsafe access, per-year caches or `LocalDateTime` resolution. The English token +index must retain UTC-first insertion, first duplicate wins, longest-first +matching and the source's case behavior. The proposed hybrid still needs broad +differential validation; source review is not proof of complete equivalence. + +Root also found that `Micros.yearMicros` is not uniformly wrapping arithmetic: +negative day counts whose product wraps positive return MIN. Near-MIN day-of-week +subtraction can itself wrap. Preserve both source helpers exactly and cover these +cases, rather than replacing them with Java Time's narrower calendar range. + +Two-digit years use the same class-initialized reference-century calculation as +the server, not a per-value clock lookup. Named-zone meanings depend on JDK +locale/tzdb data. Different process initialization windows or runtime data can +differ; matching-runtime differential tests cannot prove cross-runtime equality. +Record this limitation explicitly. No new protocol fields, version negotiation, +reduced-year contract or SQL-engine change is proposed. + +The 2.27 test preparation caught an incorrect provisional four-digit fraction +expectation before code activation: `.1234` is parsed as `.234000` seconds, +not `.123400`. The `SSz` pattern compiles as two `S` operations; the second is +greedy and overwrites the first millisecond value. This behavior was already +visible in the original public-parser probe and missed in its first summary. +The independent test author caught it when checking the proposed literal corpus. +Root verified the compiler/token explanation. Every provisional corpus row must +be checked against the actual compiled server parser before freezing; unsigned +fraction-width reasoning is insufficient. No production change has begun. + +Preparation progress: root checked 45 deterministic vectors with the actual +compiled server parser after client UTF-8 normalization; zero differences. +Seven reference-century/timezone vectors are separate runtime-dependent cases. +The fixed-width number parser is reusable from client Numbers: signs and internal +underscores are allowed; it is not a digit-only parser. Greedy numeric fields +have different rules and need narrow source copies. Parsed non-null text can +produce raw MIN with a present wire value and SQL NULL; Java-null strings instead +write the null bitmap. Do not import the typed-LONG source-null policy here. + +Root caught and corrected test-only preparation mistakes before activation: +a nano lifecycle expected 1 instead of 1000; a native VARCHAR fixture treated +its int32 offsets as int64 values; the standalone legacy encoder fixture +incorrectly assumed Sender's delta-dictionary prefix. The chosen encoder entry +point matters as much as its per-type writer. The two-character greedy year +`-1` also belongs with reference-century vectors, not fixed 1999 literals. + +The independent hybrid-zone experiment uses public server calendar helpers to +isolate the adapter: all 604 zone IDs and 34,885 deterministic probes agree, +with checksum 6947487320943606609. The deliberately wrong all-Instant control +has 512 mismatches. Root read and independently replayed the diagnostic. This +does not prove the new client parser, locale tokens or copied calendar helpers; +their final binding/encoder differential remains required. + +The fresh native STRING-to-VARCHAR baseline is measured and independently +validated before production edits: six identical-checksum 100-million-row runs, +12 copied bytes per row, fixed 4096-row resets, no new major faults or cgroup +throttle/OOM errors. The measurement is an existing-path regression reference, +not a timestamp conversion benchmark. Test-only activation and red gates follow; +production remains frozen at the accepted 2.26 binding/JAR. + +Red gates complete on that frozen implementation. Client: five tests, one +failure and three errors, all due to missing STRING-to-timestamp conversion; +the native VARCHAR/inference/designated guard passes. Server: six tests, four +missing-conversion errors; independent legacy VARCHAR wire/SQL characterization +and public designated-timestamp rejection/recovery both pass. No fixture or +compile failure remains. Saved red XML/logs are under the 2.27 client/server +target directories. Authoring the narrowly scoped parser may now proceed. + +The corpus grew to 110 literal cases, checked against the actual server parser +after client UTF-8 normalization (zero differences), plus seven runtime inputs +exercised with both units. Extreme micros are not arbitrary representable longs: +the only grammar reaching them has millisecond precision, so wrapping results +are multiples of eight. MAX-7 is reachable; MAX through MAX-6 are not. Tests use +source-proven reachable strings, saturation/wrap boundaries and exact nano +conversion limits rather than inventing impossible MAX fixtures. + +The initial candidate passed the focused client tests but failed broader public +binding/encoder comparison. Root found the missing last fixed-SSS fallback: +`.+12Z`, `.1_2Z` and `.10000` are accepted through fixed integer parsing and +numeric-zone parsing. The fix adds that attempt after the existing greedy +forms, without changing their priority. Expanded ISO-week testing then found +1297-W01 and 1697-W01 differing by one day. The server retains the originally +parsed year's leap flag after adjusting the ISO-week year; the client had +recomputed it. Removing that recomputation preserves source behavior. Both +findings now have shared literal regressions for both timestamp units. + +Validation also caught a diagnostic-fixture error: the scratch encoder reader +assumed Sender's dictionary prefix and a bitmap-length field. Standalone +encodeSchema has neither. Root rejected that run, checked the actual writer, +and required the corrected diagnostic to decode complete frames through EOF. +The failed log remains saved. An independent hour-24 review finding was withdrawn +after reading the prior normalization in the server's compute method; no +production change was made for it. Reviews are inputs, not acceptance evidence. + +Final parser hash `8c8999302508f7758f564cab5df3b12ba1d3d527206faeda88f9e1d9b1a8399d` +and binding hash `71a53f5fe783551819f5277b76767fedfb506b90c85b932ac90fbd8bd33f4d78` +are frozen for final gates. The public binding/encoder differential passes +214,030 cases in each fresh en_US/tr_TR JVM, including all four-digit ISO-week +years, named-zone aliases, transitions, fraction/sign combinations and seeded +inputs. Checksums are 16401770755196548037 and 7304463666663708165 respectively. +Locale-dependent acceptance differs as it does on the server; each run compares +matched runtime data. Focused client validation passes five tests plus two +packaging checks. Canonical, installed-server and native-path performance gates +remain pending; this is not yet an accepted conversion checkpoint. + +Final functional gates now pass 442 client tests plus two packaging checks, +47 focused server tests and 311 server regressions. The unchanged original +333-test selector reports zero failures and 28 errors; exact name comparison +removes only STRING-to-TIMESTAMP from the previous 29. Seeded recovery repeats +are still required before acceptance. + +The final native STRING-to-VARCHAR performance comparison completes all twelve +100-million-row runs with identical byte/checksum verification. Instructions +are effectively unchanged (-0.01%); basic cycles and elapsed consistently rise +about 2–3%. This passes the predeclared 5% rejection threshold but must not be +called unchanged performance. Variable pipeline metrics do not identify a +hardware cause. No new major faults, throttle/OOM events or visible kernel +warnings occurred. Root read the measurements and independently validated the +study. This guard does not measure timestamp-parser or ingestion throughput. + +Final acceptance: all three seeded repeats pass 74 tests each (1/2, 42/43, +1234567/7654321). Root verifies the selected XML seed and project-local tmp +properties after every run, the exact one-name diagnostic delta, mirrored +corpora, frozen source hashes and installed JAR. The final focused gate also +passes after strengthening public error-message context assertions. Accepted +checkpoint: 28 baseline errors remain; the wider conversion inventory and +release gates are still incomplete. Exact commands, failed diagnostics and +frozen identities are in the 2.27 client/server run manifests. + +Final independent SOL audit confirms the same counts, one-name removal, +standalone installed JAR classpath, all three seed/tmp pairs and frozen hashes. +Both run manifests are updated from pending to accepted; no review finding +remains for this slice. + +## D044 — Timestamp-to-text: approved checked normalization + +Status: accepted in iteration 2.28, 2026-09-11. The initial +preparation and approval follow chronologically. Root and SOL source audits confirm both +text targets use the server's millisecond formatter; nano source counts divide +by 1000 toward zero first. Instant's legacy source precision is micros, even +when it contains a finer fractional part. Prefer one schema-only formatter +over a general format compiler or refactoring the now-tested parser. + +Root rejected an initial null-policy finding based on the wrong cursor. +QwpTableBlockCursor always uses QwpTimestampColumnCursor for both timestamp +types, regardless of Gorilla mode. That cursor never infers null from MIN; +only the bitmap decides nullness. A standalone unflagged encoder/server-cursor +probe confirms present MICRO MIN becomes empty text, present NANO MIN becomes +1677-09-21T00:12:43.145Z, and bitmap omissions remain null. Preserve these +semantics; no new timestamp-to-text null-policy decision is needed. The probe +uses the public encoder's Gorilla-advertised, uncompressed-column fallback; +it does not claim a separately configured Gorilla-disabled Sender test. +Its literal output also distinguishes year -1 (`000-1`) from year zero/one +(`0001`); summaries of helpers are not a substitute for full formatter output. + +D021 approved range checking for timestamp targets, not text targets. Legacy +larger-unit and Instant lowering can wrap into unrelated dates. Root recommends +checked conversion into the existing source precision before formatting, while +leaving legacy mode unchanged, and has requested this narrow user decision. +Do not activate the new converter or assume approval while it is pending. +Read-only preparation is under +core/target/schema-timestamp-text.2.28-client/preparation/; it is not an accepted +conversion or a real-server ingestion test. +Root independently replays the probe against the frozen installed 2.27 client +JAR: exit zero and byte-identical literal output. The pending decision remains +timestamp-to-text overflow, not timestamp null handling. + +The user explicitly approved rejection of out-of-range timestamps written to +STRING/VARCHAR. Normalize primitive NANOS by division toward zero; convert +MICROS through DAYS with checked microsecond arithmetic. Convert Instant with +the existing final-value-safe helper (including its negative endpoint handling) +and existing microsecond rounding. Reject invalid units and null Instant as +before; ignore duplicates before conversion or argument validation. Preserve +the source timestamp MIN behavior described above. Legacy code and server/SQL +production code remain outside this increment. + +Implementation starts with public client/server test preparation and a fresh +native-timestamp performance baseline. Production remains frozen at accepted +2.27 until the baseline gate and red tests are checked. Prefer a small separate +schema formatter and reuse timestampTextSink, whose producer-owned ASCII view +can be copied directly into the existing VARCHAR buffer. Do not add another +scratch field, a compiler, send-time transformation or a new replay path without +evidence that the existing structure is insufficient. + +The pre-edit native MICRO baseline passes the hardware-study gate: six fixed +500-million-row runs have identical checksum 3807237185703477635. Basic medians +are 6.956 seconds, 76.790 cycles/row and 550.126 instructions/row. The comparison +will reject a repeatable 5% cycles/instructions increase; this is a native-path +regression guard, not a text-conversion throughput claim. + +Root validates all 63 shared literals independently against checked BigInteger +normalization and the public server formatter (checksum 13924035717609678081). +The test-only red gate reports five client tests (one failure/four unsupported +conversion errors) and four server tests (one failure/two unsupported conversion +errors); the independent legacy ingestion method passes. Native generation +encoding succeeds before the expected text rebind error. Production is now +authorized for this increment. + +Preparation review rejected test defects before implementation: an invalid UUID +setup value, a misplaced catch, and a scratch oracle/reader that incorrectly +treated empty timestamp text as NULL and invented VARCHAR framing fields. +Corrected tests and the oracle read the actual writer and cursor contracts; +failed diagnostics remain saved. Review summaries alone are not acceptance. + +The production review caught an incorrect epoch-day constant (719528 rather +than the server's 719527) and a Java constant forward reference before the +candidate gate. Both were corrected. Target validation now precedes a simple +NANO-versus-micro conversion branch; there is no repeated text-target conversion +decision or new conversion framework. The formatter has one fixed presentation +and reuses the existing producer-owned UTF-8 sink. + +Frozen candidate binding fb36c954 and formatter c99de83d pass root and independent +SOL source review. Client focused validation passes five tests plus two packaging +checks; canonical validation passes 447 plus two. Installed/workspace/archive JAR +SHA256 is bc65098cf79eb28baeeb63ab1617c6479e34ff75302f1fad750161f33139252a. +The public binding/encoder differential checks 160,202 conversions against the +independent server formatter and BigInteger range oracle: zero mismatches, +checksum 4937659523843806437. Server and performance gates remain pending. + +Focused server validation now passes 53 tests, including the new real-Sender +corpus and independent legacy MIN/bitmap ingestion. The original conversion +tests and both server appenders retain their 2.27 hashes. + +The native MICRO A/B guard passes all twelve fixed 500-million-row runs with +the baseline checksum. Basic paired cycles differ by +0.432%, -1.091%, -0.340%; +instructions are effectively equal. No repeatable regression is detected on +this path. This does not measure new timestamp-text or overall ingestion +throughput. Final server regression and repeat gates remain pending. + +The final server regression gate passes 317 tests. The literal original 333-test +selector reports zero failures and 26 errors; name-by-name comparison removes +only testTimestampToStringColumn and testTimestampToVarcharColumn from the prior +28. No new failing method appears. Three seeded recovery repeats are running +before acceptance; the broader conversion inventory remains a release gate. + +Final acceptance: all three seeded repeats pass 80 tests each (1/2, 42/43, +1234567/7654321). Root checks every run's raw result, archived seed/tmp evidence, +unchanged original failure selector and exact two-name removal. Both frozen +source manifests and installed/archive JAR remain consistent. The hardware +study independently validates. Accepted checkpoint: 26 original errors remain; +this is not full conversion or release acceptance. Exact commands and evidence +are in the 2.28 client/server run manifests. + +Final independent SOL acceptance confirms all raw totals, exact failure-name +delta, corpus/source/JAR hashes and all 48 selected repeat XML files. Root also +parses those 48 reports independently: 16 classes and 80 passing tests per run, +with exact seeds, project-local tmp and installed client JAR. Both manifests +now mark phase 3c accepted; no review finding remains for this increment. + +Next-slice preparation is read-only: phase 4a adds CHAR identity and STRING-to-CHAR, +not CHAR-to-text. Preserve first-character truncation rather than introducing +new one-character validation. Public UTF-16 replacement must match actual server +UTF-8 decoding: BMP first characters survive, supplementary-first becomes NUL, +and malformed surrogates become '?'. Reuse short storage; add the missing CHAR +wire-type mapping before activating the native setter. No new sink, formatter, +protocol parameters or conversion framework is needed. Implementation and +new tests are not yet started for this slice. + +## D045 — CHAR identity and STRING-to-CHAR + +Status: iteration 2.29 accepted, 2026-09-11. The user approved the next slice. +Root and SOL reread the server CHAR dispatch, first-character decoder, client +UTF-16 replacement, existing short storage and Sender rollback before edits. +Keep native CHAR as a two-byte code unit. STRING input takes the first BMP +character; trailing text is ignored. Empty, NUL and supplementary-first inputs +produce a present code unit zero. For malformed UTF-16, schema lowering produces +the same '?' as existing client UTF-8 replacement followed by server decoding; +schema mode writes the resulting CHAR directly, not replacement UTF-8 bytes. +Java null remains an effective bitmap-null write. CHAR's stored null sentinel +is zero, so present empty/NUL/supplementary-first inputs also read as SQL NULL. +Malformed raw UTF-8 yields zero on the legacy server path; it is not the same +case as malformed UTF-16 passed to the public string setter. + +Reuse targetColumn's exact-type helper for the native setter, add the missing +CHAR wire mapping, and use a small first-character conversion without allocating +another sink. Do not add CHAR-to-text, SQL/server changes, protocol metadata, +send-time conversion, a new replay path or a conversion framework. + +Public Sender tests provide the pre-edit red gate. A component test that calls +the new binding method is prepared outside the test source tree until that +method exists; do not add a temporary production stub merely to compile a red +test. A fresh native STRING-to-VARCHAR performance baseline precedes production +edits. Root owns serialized Maven and acceptance; SOL owns bounded test, +implementation, independent-review and performance tasks. + +The pre-edit native string baseline passes the validated hardware study: six +100-million-row runs each verify 1.2 billion bytes and checksum +15640344951909530213. Basic medians are 3.340 seconds, 186.271 cycles/row and +916.699 instructions/row. The predeclared guard rejects repeatable 5% normalized +regressions; it makes no claim about new CHAR-conversion throughput. + +Root independently validates all 23 corpus literals against the actual client +UTF-8 sink and public server first-character decoder (checksum +1837560376326485833). Native CHAR expectations come from input code units, not +the expected-result field. Client red reports two tests with one failure and +one missing-conversion error; server red reports four tests with three missing- +conversion errors and the independent legacy test passing. Production is now +authorized. Preparation review corrected circular test inputs, impossible +designated metadata, incorrect error reasons, raw frame/ACK assumptions and a +scratch reader's nonexistent CHAR encoding byte before candidate validation. + +The first focused candidate gate reports five tests with one failure: native +CHAR did not refresh a cached schema on rejection. Initial review incorrectly +attributed this to the mock fixture. Root traced the public setter and found +the missing LineSenderSchemaException catch used by every previously activated +setter. Keep the original failing generation test and add that same existing +rollback-and-refresh handling; do not weaken the test or change cache freshness. +The initial source acceptance is superseded pending this correction and rerun. + +The corrected Sender (4904b4ef) passes the original CHAR-triggered refresh test; +independent review confirms the standard catch is now present. Binding remains +09170988. Focused client validation passes five tests plus two packaging checks. +The frozen candidate JAR is +edb2a9dbae3ba64d0ab3caf07309d4755187eabbfe676ad7a1fc36aa43642c6a. +Root's public binding/encoder differential checks 65,549 STRING inputs against +the actual client UTF-8 sink and public server decoder: zero mismatches, +checksum 13914181573124632075. Performance and remaining regression gates are +still pending; this is not acceptance. + +The existing native STRING-to-VARCHAR hardware guard passes all twelve fixed- +work A/B runs. Paired basic cycles change by +0.659%, +0.795%, +0.114%, with +instructions effectively unchanged and identical output checksums. No +repeatable regression reaches the declared 5% guard. This is not a CHAR +conversion throughput measurement; full functional validation remains pending. + +Client canonical validation passes 452 tests plus two packaging checks. Install +is green; workspace, installed and archived candidate JAR hashes are identical. +Root reruns the hardware-study validator successfully. Server validation now +uses that exact installed artifact, with no overlapping client rebuild. + +The first focused server candidate run reports 59 tests with one test-query +error: the inferred-table assertion used `ts`, but auto-creation names the +designated column `timestamp`. Both writes and ACKs succeeded, as did the two +original CHAR conversion methods. Correct only the test query/header; preserve +the failed run as evidence. No server or SQL production code changes. + +With the inferred-column assertion corrected, all 59 focused server tests pass. +The final server fixture hash is db3cbf8c. The wider regression gate passes +323 tests. The original 333-test selector is unchanged; its diagnostic and +three seeded repeats are still pending. + +The unchanged original 333-test diagnostic now reports zero failures and 24 +errors. Root compares method names against 2.28: only testCharToCharColumn and +testStringToCharColumn are removed, with no additions. Three seeded repeats +are running before final acceptance. + +Final root acceptance: all three seeded repeats pass 86 tests each (1/2, 42/43, +1234567/7654321). Root parses all 51 selected XML reports independently: 17 +classes per run, exact seeds and project-local tmp, standalone installed client +JAR, no failures/errors/skips. Source/corpus/JAR hashes remain frozen. The +accepted checkpoint has 24 original errors remaining, not full release coverage. +Exact commands, raw logs and hashes are in the 2.29 client/server run manifests. + +Read-only next-slice preparation confirms STRING-to-LONG256 can use the existing +32-byte append storage and four local longs; do not add a retained scratch +object or activate native LONG256. Root and SOL checked Numbers.extractLong256, +parseLong256, Long256FromCharSequenceDecoder and putStringToLong256Column: +lowercase 0x, 2–64 hex digits in complete pairs, right-to-left limbs, and rejection +of the four-MIN-limb literal. Java null/omission remains null. These are existing +server rules, not a new policy choice. No implementation has started for 4b. + +Final independent SOL acceptance confirms all raw totals, the exact two-name +failure reduction, all 51 selected repeat reports, frozen source/JAR/corpus +hashes and revision 47's bounded acceptance claims. No review blocker remains. + +## D046 — STRING-to-LONG256 + +Status: iteration 2.30 accepted, 2026-09-11. The user approved proceeding. +Root and SOL reread the server target dispatch, both extractLong256 overloads, +parseLong256, the four-word decoder and the concrete WAL appender. Preserve the +lowercase 0x prefix, complete byte pairs, 2–64 case-insensitive hex digits and +right-to-left limbs. Java null/omission remains null; a non-null literal with +four Long.MIN_VALUE limbs is INVALID_VALUE, as on the server. No policy change. + +Add only the STRING target arm, LONG256 wire mapping and a narrow append helper +using four local longs and existing parseHexLong/addLong256. No retained scratch +object, native LONG256 setter activation, schema state, send-time conversion, +server production or SQL-engine change. Existing stringColumn already has the +typed rollback/refresh handling missing from CHAR's initial activation. + +SOL prepares shared literal vectors, exact public-wire component tests, public +Sender rollback/generation tests and real-server/independent legacy E2Es. Root +owns serialized red, regression, install and seeded repeat gates. A fresh native +STRING-to-VARCHAR baseline and validated hardware study precede production +edits; it is a regression guard, not a LONG256 throughput claim. + +Fresh pre-edit measurements pass the hardware study, independently validated +by root. Six 100-million-row runs verify 1.2 billion bytes and checksum +15640344951909530213 each. Basic median cycles/row 184.538 and instructions/row +916.792; no major faults or new throttle/OOM events. Baseline JAR edb2a9db and +binding 09170988 are archived before production changes. + +Pre-red root review catches two test assumptions: parsing hex as LONG yields +INVALID_VALUE, not UNSUPPORTED_FEATURE; a cached VARCHAR accepts arbitrary +strings and cannot trigger rejection-driven refresh. The generation test will +instead retain a LONG256 block, switch to VARCHAR and use an invalid LONG256 +string to trigger refresh through the same public setter. Keep the existing +cache policy; correct the test sequence, not production. + +Client red confirms six tests with four missing-conversion errors and no +failures. Root independently checks the 23 shared literals against public +server parsing: 10 values, one Java null, 12 invalid, zero mismatches (checksum +12475823559777660557). Review corrected a scratch oracle that had mistaken the +NULL marker for literal invalid text; it now checks actual Java null and exact +bitmap-only encoding and surfaces unexpected parser failures. + +The initial server red has two missing-conversion errors and one legacy fixture +failure. Correct the raw column count to include the unnamed timestamp, and +retain whole-byte padding in the expected hex text. These are test corrections; +the legacy oracle must pass before production edits. The first red log/XML is +retained separately. + +The corrected server red reports three tests with two missing-conversion errors +and the independent legacy test passing, including rejection of odd-length and +four-MIN literals. Root authorizes the Binding-only production change after +both behavioral red gates, the corrected literal oracle and the validated +pre-edit performance study. Existing Sender lifecycle code remains unchanged. + +Independent source review accepts the bounded parser logic. The first focused +build catches the existing error helper's boxed Integer parameter: new calls +must cast the short ColumnType constant to int, as neighboring helpers do. +Correct only those calls and rerun before freezing a candidate artifact. + +The corrected Binding (a57ce9ae) passes six focused client tests plus two +packaging checks; independent source review confirms only the two casts changed. +Frozen candidate JAR: +37169edc56700c7e34ec357d0eb55d8b25886731faf8a65491f7d80fdd4a270a. +Root's public binding/encoder differential completes 40,055 cases with zero +mismatches against actual client UTF-8 encoding and public server parsing. +Performance A/B and broader client/server gates remain pending. + +The native STRING-to-VARCHAR A/B guard passes all twelve fixed-work runs with +matching output checksums. Paired cycles change by +1.271%, +1.152%, +0.055%; +instructions change by about -0.01%. No repeatable change reaches the declared +5% regression threshold. This does not measure STRING-to-LONG256 throughput. +Broader functional validation now resumes with the frozen candidate. + +Client canonical passes 458 tests plus two packaging checks, and install passes. +Workspace, installed and archived JAR hashes match 37169edc. Root independently +reruns the final hardware-study validator successfully. Real-server validation +now uses this exact installed artifact with no overlapping client rebuild. + +All 63 focused server tests pass, including the original STRING-to-LONG256 +method, shared value/null/invalid corpus, whole-row recovery and independent +legacy value/null/omission/rejection checks. The broader server gate is running. + +The wider server regression gate passes 327 tests. The original 333-test +selector is unchanged and now running; final acceptance still requires its +exact failure-name comparison and three seeded recovery repeats. + +The unchanged original 333-test diagnostic reports zero failures and 23 errors. +Root compares names against 2.29: only testStringToLong256Column is removed; +there are no additions. Three seeded recovery runs remain before acceptance. + +Final root acceptance: all three seeded repeats pass 90 tests each (1/2, 42/43, +1234567/7654321). Root independently parses the 54 selected XML reports: 18 +classes per run, exact seeds and project-local temporary storage, standalone +installed client JAR, zero failures/errors/skips. Frozen production/test/corpus +hashes remain unchanged. Client canonical is 458 plus two packaging checks; +server focused is 63, canonical 327, and the original diagnostic remains +333 tests with 23 errors. Revision 48 and the remaining-failures plan record +phase 4b acceptance, not release acceptance. Exact commands, raw logs, hashes +and the selected-report audit are in the 2.30 client/server run manifests. + +Read-only preparation for phase 4c traces the server's string-to-geohash parser, +cursor and target precision checks against the existing client buffer/writer. +Precision is already packed in the full schema type; no protocol parameter is +needed. A null-only column still needs precision initialized without a value +append. Empty text takes the present all-ones path, whereas Java null uses the +bitmap; both read as SQL NULL. Keep this distinction in exact wire tests and +reuse the existing precision field. Native geohash setter activation remains +separate work. No phase 4c implementation has started. + +Final independent SOL acceptance confirms all 54 selected repeat reports, +raw totals, exact seeds/tmp/installed-JAR provenance and frozen fixture/corpus +hashes. Revision 48, the plan and D046 consistently bound acceptance to phase +4b with 23 original errors remaining. Native LONG256 remains unimplemented; +performance evidence is only the native STRING-to-VARCHAR regression guard. +No review blocker remains. + +## D047 — STRING-to-GEOHASH + +Status: iteration 2.31 accepted, 2026-09-11. The user approved continuing. +Scope is phase 4c only: string conversion to existing GEOHASH targets using +their packed schema precision. Native geohash setters and decimals stay separate. +Root delegates client tests/implementation, server E2Es, and performance plus +adversarial review to SOL. Root owns serialized Maven/install gates and records +decisions. No server production or SQL-engine change is planned. + +Revalidation corrects D046's read-only next-slice recommendation. Server VARCHAR +conversion maps empty text to the target's -1 null sentinel at every precision. +But native GEOHASH reads only ceil(bits/8) bytes: without a bitmap, a valid +all-ones value at byte-aligned precisions becomes null; with a bitmap, a packed +empty sentinel can zero-extend to a non-null value in larger target storage. +Root and SOL independently traced QwpGeoHashColumnCursor.advanceRow/readValue, +WalColumnarRowAppender.putStringToGeoHashColumn/putGeoHashColumn and the type's +storage-width calculation. Source presence cannot simply be copied to target +presence. The earlier claim that native present -1 always stores as null was +wrong. Add real-server characterization before production edits. + +Recommended minimal representation: empty text uses addNull, and schema-directed +GEOHASH columns always emit the explicit null bitmap, even when all rows have +values. A private writer argument can distinguish the existing schema/legacy +encoding entrypoints without new retained state or scanning values. Cost is one +bit per row; legacy encoding stays unchanged. This is null framing, not a +send-time value conversion. Initialize precision through the existing column +field, factoring the current addGeoHash validation. Preserve full type flags +and existing timestamp handling; do not globally switch all dispatch to tagOf. + +The hardware-first performance skill requires a fresh pre-edit regression +baseline for the shared string/buffer/writer path. SOL has completed six fixed +100-million-row native STRING-to-VARCHAR runs with matching 1.2-billion-byte +checksums; the validated study and behavioral red gates still precede production +authorization. This guard does not measure GEOHASH conversion throughput. + +Root independently validates the pre-edit study and all 18 shared literal +vectors against actual client UTF-8 plus the current server's public parser: +13 values, two nulls, three invalid, zero mismatches. The initial client red +reports four missing-conversion errors and no failures. Review then corrects +test-only assumptions before the final red: at eight-bit precision `04`, not +`01`, truncates to 1; ordinary encode(table) has no symbol-delta prefix. Expand +permanent tests to all 60 precisions rather than only one 60-bit target. + +Two boundary checks refine the brief without a new abstraction. First, the +existing append benchmark does not call the encoder; extend the pre-edit +measurement to cover the shared writer separately. Second, the low-level +schema encoder accepts manually constructed non-nullable buffers whose bitmap +address is zero. A forced bitmap must be limited to existing nullable storage; +schema bindings always use that storage. Keep manual sentinel-mode encoding +unchanged and test it through the public encoder, avoiding a new allocation or +a null-address copy. + +The final corrected client red reports five missing-conversion errors, no +failures. Server red reports three tests with two missing-conversion errors; +the independent legacy VARCHAR and native GEOHASH characterization passes. +This confirms both packed-width traps at runtime before production changes. +Root also validates the extended two-card performance study: separate append +and encodeSchema baselines, three basic and three pipeline runs each. The +encoding runs verify checksum 114033733800999820 with zero major faults. + +Root authorizes the narrow production change after these gates. The existing +ColumnBuffer.usesNullBitmap() getter expresses the nullable-storage condition +directly; reuse it rather than inspecting pointers or adding a field. Keep +the current full-type switches for all non-geohash targets. Review, focused +green, regressions, A/B, original failure-name delta and repeats remain ahead. + +The first candidate passes five focused client tests plus two packaging checks, +and 7,210 public differential cases across all precisions (6,870 accepted, 340 +rejected, zero mismatches). However, the full 463-test client gate finds three +real regressions: existing flagged STRING/BOOLEAN/LONG schema types are mistaken +for geohashes and produce an untyped zero-precision error. Source-only acceptance +is withdrawn. Preserve these existing tests; recognize a GEOHASH target only +when its precision is 1–60 and its full type exactly matches the canonical +packed type for that precision. Unknown flags/tags must keep the existing typed +UNSUPPORTED_FEATURE path. No generic exception wrapper or new schema state is +needed. Candidate 41722b7b and the failed full-suite evidence are archived as +initial, not accepted; A/B remains on hold until the corrected candidate. + +The server recovery fixture also had an insufficient one-character zero at +eight-bit precision. Change it to `00`, assert the failed row's INVALID_VALUE, +and assert exact A/C/D values and nullness. The red-tested fixture was not copied +before this edit; its archive was reconstructed by reversing exactly those +three edits and verifying the original SHA-256 77ab62ac. Current fixture SHA is +6bd7f662. This is an exact reconstruction, not a pre-edit capture. + +The corrected Binding (b85add96) recognizes only complete canonical GEOHASH +types. The original three regression tests are unchanged; new malformed-type +vectors add zero/61-bit, wrong-tag and unknown-flag coverage. Six focused client +tests plus two packaging checks pass. Independent SOL source review accepts +the correction. Frozen candidate JAR: +c75a826aeb85e9e5f3728d8bd4284c8cd62aab18b8cc3d3c9829e686cc4396c4. +Root reruns the 7,210-case public differential against current server classes: +6,870 accepted, 340 rejected, zero mismatches, checksum 11748646508706949247. +The full client gate is rerunning before install and A/B. + +The corrected full client gate passes 464 tests plus two packaging checks. +Install passes, and workspace/archived/installed JARs all match c75a826a. +The two-path A/B window now runs exclusively, with no overlapping Maven work. + +All 24 counterbalanced A/B runs pass the declared repeatable 5% guard, with +matching fixed-work checksums. The cost is small but measured: native string +append instructions increase about 0.437%, cycles 1.457–1.740%, elapsed +1.461–1.799%; schema encoding instructions increase about 0.472%, cycles +0.131–1.091%, elapsed 0.166–1.148%. Do not describe this as unchanged performance +or infer GEOHASH/ingestion throughput. Root independently validates the final +two-card study. No new throttle/OOM/major-fault or visible kernel-error evidence. + +The final installed candidate passes all 67 focused server tests, including the +original STRING-to-GEOHASH case, all 60 target precisions, null-only following +batches, invalid row recovery and the independent legacy/native characterizations. +The 331-test server regression gate is running before the unchanged original +diagnostic and three seeded repeats. Frozen source/test/corpus hashes still match. + +The wider server gate passes all 331 tests. The unchanged original 333-test +diagnostic is now running; acceptance still requires its exact failure-name +comparison and three seeded recovery repeats. + +The unchanged original 333-test diagnostic reports zero failures and 22 errors. +Root compares raw failing method names against 2.30: only +testStringToGeoHashColumn is removed, with no additions. The remaining original +cases are all decimal conversions. Three seeded recovery repeats are running. + +Read-only phase 5a preparation recommends all three typed decimal inputs as one +coherent increment, with exact target precision/scale and existing wire widths. +Root rejects a second binding-owned Decimal256 scratch: ColumnBuffer already +owns one that can normalize a copy, validate it and append atomically. Preserve +public decimal-null no-op behavior and caller input objects. Internal decimal +word order and wire order differ and need exact independent assertions. +The next slice must also preserve target scale across reset and failed-row +cleanup when an emitted column is later all-null through omission, while legacy +columns retain their first-input-scale-per-batch behavior. The remaining-failures +plan records these tests; no decimal implementation is included in 2.31. + +Final root acceptance: all three seeded repeats pass 94 tests each (1/2, 42/43, +1234567/7654321). Root independently audits all 57 selected XML reports: 19 +classes per run, exact seeds and project-local temporary storage, standalone +installed client JAR and zero failures/errors/skips. Production, server fixture +and corpus hashes remain frozen. A final whitespace-only cleanup in the new +client component test is followed by a fresh six-test plus two-packaging-check +run; the client JAR remains byte-identical. The earlier tested source hash is +preserved in validated-tests-before-formatting.sha256. + +Revision 49 and the plan accept phase 4c, with 22 original decimal errors still +remaining. They do not imply complete conversion coverage or release readiness. +Exact commands, raw logs, hashes and selected-report audit are in the 2.31 +client/server run manifests. No commit, server production or SQL-engine change. + +Final independent SOL acceptance confirms all 57 selected repeat reports, +seeds/tmp/installed-artifact provenance, unchanged final JAR and bounded revision +49 claims. It reruns the hardware-study validator and verifies the measured +cost disclosure, native-setter exclusion, 22 remaining errors and final +whitespace-only test check. No review blocker remains. + +## D048 — Native decimal-to-decimal conversion + +Status: iteration 2.32 and phase 5a accepted, 2026-09-11. Revision 50 records +the bounded scope: Decimal64, Decimal128 and Decimal256 public inputs now reach +DECIMAL8/16/32/64/128/256 targets. These are 18 logical pairs over nine physical +source-width/target-width paths because the three small target types use +DECIMAL64 wire storage. Phases 5b–5e remain; this is neither complete conversion +coverage nor release readiness. + +The implementation reuses the ColumnBuffer-owned Decimal256 scratch. It copies +the caller value, applies exact rescaling, checks declared target precision and +the selected physical storage width, then appends direct target-width limbs. +Precision loss, scale-up overflow, target-precision overflow and storage-width +overflow fail locally without mutating the caller. The target scale survives +buffer reset and failed-row cleanup. Exact component and public-wire assertions +cover high-to-low internal limbs versus low-to-high wire order, row rollback, +completed-row preservation and adoption of a later schema generation. + +Public typed-decimal null/sentinel inputs retain their established no-op behavior. +Confirmed MISSING columns and legacy mode keep source-width inference and the +legacy first-input-scale-per-batch rule; schema-directed known targets do not. +Server characterization confirms decimal cursor nullness is bitmap-only: a set +bitmap bit is null, while raw sentinel limb patterns without a bitmap remain +supplied values entering conversion. This characterization does not activate +decimal text output or the LONG/STRING/FLOAT/DOUBLE decimal inputs. + +The shared client/server corpus has 60 data rows: 45 accepted values and 15 +invalid values, covering all 18 logical pairs. Both copies have SHA-256 +`3819694736f54de4d2f2f396b09ff487667051c6045b34cc6d7579552551d11d`. +Client focused validation passes eight tests plus two packaging checks; canonical +validation passes 472 tests plus two packaging checks. Frozen candidate JAR +SHA-256: `1e814a2f82836b5c9ac04cf7b5674e56f1b939440df486ae6628ebb1b8eebb08`. + +Server focused validation passes seven tests and the canonical gate passes 338. +The unchanged original 333-test diagnostic reports zero failures and 18 errors: +only the four intended decimal-to-decimal methods disappeared from the prior 22, +with no additions. All three seeded repeats pass 101 tests each for seeds 1/2, +42/43 and 1234567/7654321. Root audits 20 archived XML reports per repeat with +zero failures, errors or skips. + +The legacy-path regression guard passes all 12 balanced A/B runs with matching +fixed work and checksums. Paired instructions change by +0.001% to +0.002%; paired +cycles range from -0.47% to +1.99%, below the declared repeatable 5% rejection +threshold. A separate candidate-only composite covers all nine physical width +pairs in six runs of 20 million rows each: 180 million conversions and 3.36 GB +verified per run. Its basic medians are 32.91 ns, 182.03 cycles and 1140.18 +instructions per value. This has no semantically matched schema baseline and is +absolute-cost evidence only; it supports no network, ingestion or general- +throughput claim. One 200-million-row sizing run and an interrupted follow-on +are excluded from the accepted measurements. + +Residual P3: a scale-up overflow has the correct `INVALID_VALUE` reason and row +rollback, but its detail currently says `decimal value cannot be rescaled +exactly` instead of identifying overflow. Keep this as diagnostic wording debt; +do not add another hot-path branch solely to refine the message. + +Final acceptance is limited to phase 5a. The remaining 18 original diagnostic +errors are phase 5b LONG-to-decimal, 5c STRING-to-decimal, 5d floating-to-decimal +and 5e native-decimal-to-text. No commit, server production or SQL-engine change +is included in this documentation update. + +## D049 — LONG-to-decimal conversion + +Status: iteration 2.33 and phase 5b accepted, 2026-09-11. Revision 51 records +the bounded scope: public `longColumn` now reaches every +DECIMAL8/16/32/64/128/256 target in schema mode. Phase 5a remains the decimal +foundation; phases 5c–5e remain. This is neither complete conversion coverage +nor release readiness. + +Root and the SOL contract reviewer read the server's LONG dispatch, decimal +rescale/range helpers, decimal cursor and physical target selection before the +change. The contract is scale-zero signed input followed by exact multiplication +for a positive target scale, declared-precision validation and physical-width +validation. Small target classes use DECIMAL64 wire storage. Duplicate +suppression precedes conversion. Confirmed MISSING keeps inferred LONG. + +`Long.MIN_VALUE` needs an explicit schema-mode rule because legacy fixed-width +sentinel handling changes when another row creates a null bitmap. The accepted +rule is stable source null: for a known decimal target it marks the decimal null +bitmap and first locks the target scale. A raw-frame real-server test preserves +and documents the legacy difference: without a bitmap the sentinel is null; +with a bitmap it is supplied as a value and can either fit or fail by target +precision. No legacy behavior was changed. + +The implementation adds no Sender, protocol or server production state. It +initializes the ColumnBuffer-owned Decimal256 scratch from LONG at scale zero, +then shares phase 5a's exact rescale, precision/storage check and direct-width +append tail. A separate null helper only locks target scale before `addNull()`. +The common numeric-target predicate was deliberately not broadened because that +would silently admit decimal targets for unrelated source families. + +The shared client/server corpus has 39 rows: 19 values, 14 invalid values and +six source-null cases across all six target classes. Both copies have SHA-256 +`d9a47432a14583f8675b0035ab10c3ef9f36e78c9f4aab6a8634958eb5aafcb1`. +An independent arithmetic oracle confirms every outcome, SQL value and unsigned +low-to-high wire limb. Component tests assert target type, bitmap, target scale, +limbs, duplicate behavior, omission, whole-row rollback and reset. Public +Sender E2Es assert exact stored values and A/failed-B/C recovery. + +The first schema-generation fixture was wrong: its proposed new LONG value was +also valid under the old DECIMAL(18,4) schema, so no refresh was justified. +Root replaced it with a value invalid under DECIMAL(18,4) but valid under +DECIMAL(20,5). The corrected test proves that the sealed old DECIMAL64(scale 4) +block remains unchanged while the later block uses DECIMAL128(scale 5). This +was a test-only correction; no production response to the false assumption was +added. A first raw legacy fixture also omitted designated timestamps; adding +them made the frame valid without changing the behavior under test. + +Red evidence uses the frozen phase-5a client JAR. The four original +LONG-to-decimal methods all reject the missing conversion; the new public +fixture's two acceptance tests fail while its independent raw legacy +characterization passes. Final client focused validation passes three tests +plus two packaging checks; canonical validation passes 475 tests plus two +packaging checks. Install passes, and workspace, archived and installed client +JARs all have SHA-256 +`41d137a1c2388ed21f776e372771785377d91c15a887d94f04e82c97f505d403`. + +Final server focused validation passes seven tests and the canonical gate +passes 345. The unchanged original 333-test diagnostic reports zero failures +and 14 errors: exactly the four intended LONG-to-decimal methods disappeared +from phase 5a's 18, with no additions. Three seeded repeats pass 108 tests each +for seeds 1/2, 42/43 and 1234567/7654321; their selected XML reports preserve +the exact seed, project-local temporary directory, installed client artifact +and zero failures/errors/skips. No server production or SQL-engine file changed +for this increment. + +The performance check targets the changed common LONG binding path, not decimal +ingestion. A pre-edit fixed-work study and 12 balanced control/candidate runs +each process 500 million decoded known-schema LONG-to-LONG writes with identical +checksum `2715042977495835685`. Paired instructions change by +0.003% to +0.004%; +paired cycles change by -0.164% to +2.237%, below the declared repeatable 5% +guard. All measured runs have zero major faults and 100% counter coverage, with +no run-window cgroup throttle/OOM delta. The hardware-study validator passes. +This supports only that no material regression was detected in LONG-to-LONG; +it makes no decimal, network, ingestion or general-throughput claim. + +Final acceptance leaves 14 original diagnostic errors: STRING-to-decimal, +FLOAT/DOUBLE-to-decimal and native-decimal-to-text. No commit or push is part of +this checkpoint. + +## D050 — STRING-to-decimal conversion + +Status: iteration 2.34 and phase 5c accepted, 2026-09-11. Revision 52 records +the bounded scope: public `stringColumn` now reaches every +DECIMAL8/16/32/64/128/256 target in schema mode. Phases 5d–5e remain; this is +neither complete conversion coverage nor release readiness. + +The server parser and all six decimal target paths were read before integration. +For a known decimal target, the client parses with that target's precision and +scale, `strict=false` and `lossy=false`, preserving the server grammar's exponent +underscores, then emits the QWP decimal wire type for the target storage class; +DECIMAL8/16/32/64 share QWP's DECIMAL64 wire type. The shared client +parser now uses the current server's leading-zero and full-scale precision +calculation. This broad precision-metadata correction required updating 19 stale +assertions in 16 existing tests; the parsed values and scales are unchanged. + +The stable schema-mode null rule is independent of target width: Java null and +exact optionally signed `NaN` or `Infinity`, after supported suffix processing, +emit target-scale bitmap SQL null. Malformed special-value prefixes and extreme +nonzero integer-boundary exponents fail locally as `INVALID_VALUE`. This +avoids the server's small-decimal special-null narrowing bug. The exact-special, +precision and exponent corrections are global shared-parser changes, not only +schema-mode behavior: direct `Decimal64`, `Decimal128` and `Decimal256` string +parsing now rejects malformed special prefixes and reports extreme nonzero +exponents as `NumericException`. Confirmed MISSING and legacy Sender paths remain +VARCHAR and use server-side conversion. + +The implementation reuses the existing per-column Decimal256 scratch and the +phase-5a precision/storage/direct-width append tail. Exact special checking is +corrected centrally according to the parser's documented contract. No separate +schema parser, conversion registry, retained state, protocol, server production, +SQL or send-time transformation was added. + +The shared client/server corpus has 55 rows: 19 values, 24 invalid inputs and +12 nulls. Both copies have SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`. +Client red validation first failed eight tests. Final focused validation passes +eight tests plus two packaging checks, all-decimal validation passes 463 tests, +and the canonical gate passes 941. Candidate, archived and installed JARs have +SHA-256 +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +A 3,601,944-case differential passes ordinary server parity and records 72 +deliberate malformed-prefix corrections. + +Server focused validation passes seven tests and the canonical gate passes 352. +The unchanged original 56-test conversion class reports ten errors: exactly the +four STRING-to-decimal methods disappeared from phase 5b's 14, with no additions. +Three seeded repeats pass 115 tests and 22 selected XML reports each. + +The performance check targets the changed native STRING binding path, not +decimal ingestion. Twelve balanced fixed-work native STRING-to-VARCHAR runs have +identical checksum `15640344951909530213`. Paired basic instructions change by +-0.001% to -0.000% and paired cycles by -1.028% to -0.007%, with no repeatable +5% regression. This makes no decimal, network, ingestion or general-throughput +claim. + +Final acceptance leaves ten original errors: FLOAT/DOUBLE-to-decimal and +native-decimal-to-text. No commit or push is part of this checkpoint. + +## D051 — FLOAT/DOUBLE-to-decimal conversion + +Status: iteration 2.35 and phase 5d accepted, 2026-09-12. Revision 53 records +the bounded scope: public `floatColumn` and `doubleColumn` now reach every +DECIMAL8/16/32/64/128/256 target in schema mode. Phase 5e remains; this is +neither complete conversion coverage nor release readiness. + +Root and two SOL reviewers traced the real server route before integration. +`WalColumnarRowAppender` reads FLOAT as a Java float and widens it to double; +both source widths then call `Numbers.doubleToDecimal(value, precision, scale, +false)`. That path formats the shortest round-tripping decimal and rejects a +natural scale above the target instead of rounding or truncating. Target +precision overflow also rejects. All six server target loops map NaN and both +infinities to decimal null. SQL casts use a different lossy setting and were not +used as the oracle. + +The client follows that contract directly. FLOAT widens through the existing +double-valued binding path. Finite values reuse the schema-only Ryu formatter, +the shared decimal parser at the target precision/scale, the existing per-column +Decimal256 scratch and phase 5a's direct target-width append tail. Small decimal +targets use DECIMAL64 wire storage while retaining their declared precision and +scale. Non-finite values use the target-scale decimal bitmap-null helper. +Duplicate suppression still precedes conversion, and failures remain +nonretryable `INVALID_VALUE` errors that permit whole-row rollback. Confirmed +MISSING and legacy mode keep native FLOAT/DOUBLE tags and server-side conversion. +No parser, state, conversion registry, protocol, server production, SQL or +send-time transformation was added. + +The first correct implementation placed the decimal branch in the common +floating numeric helper. Functional tests passed, but the hardware guard found +a real native-path cost: the helper grew from 311 to 411 bytecodes, exceeded +HotSpot's hot inlining threshold and increased instructions by about 4.7% for +DOUBLE-to-DOUBLE and 3.8% for FLOAT-to-FLOAT. The implementation was simplified +before acceptance: numeric targets retain their old branch and bytecode shape, +while decimal dispatch lives at the top of the existing cold nonnumeric helper. +No production guard or abstraction was added. + +The final fixed-work A/B study contains six counterbalanced control/candidate +pairs for each native path, 500 million writes per run. DOUBLE checksum is +`11101713426464862245`; median paired elapsed, cycles and instructions changes +are -1.36%, -1.04% and +0.0037%. FLOAT checksum is `3646854813611232293`; +medians are -0.95%, -0.83% and +0.0001%. All 24 runs have identical work and +checksums, zero major faults, complete counters and no cgroup throttle/OOM delta. +Both hardware-study validators pass. This supports only that no material native +binding regression was detected; it is not decimal, allocation, network, +ingestion or general-throughput evidence. + +The server-generated client/server corpus has 60 rows: 24 FLOAT and 36 DOUBLE; +37 values, 17 invalid values and six nulls. It covers all twelve source/target- +class pairs, exact wire width, scale and limbs, no-rounding boundaries, widened +float `0.1`, signed zero, non-finite values, `Float.MIN_VALUE` scale 60/59, +underflow and extreme overflow. Both copies have SHA-256 +`366a889719c6e7b688ff00e1bdfb95e273ab192508a1094dac5312c3d899611d`. +The generator calls the current server `Numbers.doubleToDecimal(..., false)`; +its output is byte-identical to both resources. An independent differential +checks 5,996,868 finite target/value combinations and matches acceptance, scale +and all four limbs. + +Component tests consume every vector and assert schema flag, physical target +type, bitmap, target scale, limbs and frame length. A second test covers +duplicate-first-wins, non-finite-null-first, omission, reset and A/failed-B/C +rollback. The public socket test changes DECIMAL64(18,4) to DECIMAL128(20,5): +the old block stays pinned while a value invalid under the old snapshot causes +one refresh and succeeds under the new snapshot. Final focused client validation +passes 63 tests plus both packaging checks. The corrected canonical gate passes +944 tests across 46 classes. Candidate, archive and installed JAR SHA-256 is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +The final binding source SHA-256 is +`2f8e23ba68fb415a8ff9ca7b5535c9f27d040a7b9d39541212baec0310efa1ea`. + +Four real-server E2Es consume the same corpus through public Sender, assert exact +SQL/null results and local typed errors/recovery, preserve native inference for +a missing table, and independently send an unflagged raw legacy frame with and +without null bitmaps. They inspect native tags, raw bits and bitmap context before +the server performs its old conversion. A separate server rebind or store-and- +forward fixture was not added: the new client socket test already pins the +width/scale generation, while existing decimal and schema replay tests exercise +persisted direct decimal bytes. Adding another scenario would repeat machinery +without exposing a phase-specific behavior. + +Server focused validation passes the four new E2Es and four recovered original +methods. The canonical gate passes 360 tests. The unchanged original 56-test +conversion diagnostic has zero failures and six errors: exactly the four phase- +5d methods disappeared from phase 5c's ten, with no additions. The remaining six +are the phase-5e native-decimal-to-STRING/VARCHAR methods. Three seeded repeats +pass 123 tests each for seeds 1/2, 42/43 and 1234567/7654321; each archived +selection has 23 XML reports with the requested seeds, project-local temporary +directory and zero failures/errors/skips. The installed JAR hash remained the +accepted candidate throughout server validation. No server production or SQL +engine file changed for this increment. + +Two failed validation setups are retained but excluded. The first 60-table +server run used the nearly full shared `/tmp`; WAL mmap creation hit errno 28 on +the last two tables. The complete project-local-temp rerun passed all four tests. +The first client canonical reconstruction incorrectly read every stale XML file +in the phase-5c evidence directory instead of the 45 classes actually named in +its log. That exploratory run executed 1,278 tests and exposed four errors in +`CursorWebSocketSendLoopErrorLatchTest`: the test used +`Unsafe.allocateInstance`, skipped the real final schema-coordinator initializer +and then invoked private `recordFatal` by reflection. Adding a production null +guard would have served only this impossible object state. The obsolete 251-line +test was deleted after SOL review; seven public/real-loop replacement classes +pass 12 tests, and the expanded 79-class regression gate passes 1,269 tests. +The exact prior-log-plus-new canonical selector then passes 944. This cleanup is +not evidence for the conversion itself, but it preserves the design rule against +reflection fixtures and avoids defensive production code. + +The review identified one sibling fixture with the same bare-loop pattern. +`CursorWebSocketSendLoopJvmErrorTest` left the engine null, so its private +`ioLoop` case entered a zero-backoff retry and the isolated run timed out after +60 seconds with millions of attempts. The legitimate public +`connectWithRetry` test remains; only the two reflective private-loop cases and +their fake-object plumbing were removed. The retained test passes. A normally +constructed real-loop test now covers one-attempt JVM-error escape, producer- +visible terminal wrapping and clean I/O-thread shutdown without reflection or a +test hook. No production change was made for this impossible object state. + +Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD +remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with the recorded +dirty worktrees. Final acceptance leaves six original diagnostic errors, all in +native-decimal-to-text phase 5e. No commit or push is part of this checkpoint. + +## D052 — Native-decimal-to-text contract and malformed-wire boundary + +Status: phase 5e prepared, not yet accepted, 2026-09-12. Revision 54 records the +contract before production code changes. The remaining six original failures +are `Decimal64`, `Decimal128` and `Decimal256` into STRING and VARCHAR. + +The server target dispatch promotes each native wire width to its reusable +Decimal256 value, applies the unsigned source wire scale and calls `toSink`. +The output is plain fixed-point text with valid source scales and trailing zeros +preserved. Schema mode will therefore promote a non-null public decimal into a +reused binding-owned Decimal256, apply the same unsigned one-byte scale, format +it into the reused text sink and append the existing `TYPE_VARCHAR` +representation for both text targets. MISSING and legacy mode remain native +decimal. Public null and `NULL_VALUE` inputs retain their existing no-op +semantics and do not claim a duplicate slot. No protocol, server, SQL, +buffer-state, conversion-registry or send-time change is justified. + +The compatibility boundary is explicit. A raw non-bitmap DECIMAL64/128 sentinel +is currently formatted by the server as the physical minimum integer; the raw +DECIMAL256 sentinel tuple produces an empty non-null string. Public decimal null +sentinels remain setter no-ops. Public low-level constructors and `setScale` +mutators can, however, construct other coefficients and scales outside the +documented 18/38/76-digit domains. The first implementation incorrectly called +each narrow concrete formatter directly; a full-width Decimal128 value was +silently truncated because the server formats it only after Decimal256 +promotion. That implementation is rejected. The corrected path mirrors the +server's promotion and unsigned one-byte scale instead of adding a new local +rejection policy. Hardening the Decimal API or server validation would be a +separate production decision. + +The frozen phase-5d client artifact is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +Against it, the unchanged real-server diagnostic fails exactly the six phase-5e +methods with unsupported conversion errors. One earlier setup failed before +tests because its exact project temporary directory did not exist; that log is +retained as invalid setup evidence and is excluded. Shared corpus, component, +public Sender and independent legacy-frame tests must be red against this frozen +artifact before the narrow binding branch is implemented. + +## D053 — Promote native decimals before text formatting + +Status: accepted in iteration 2.36, 2026-09-12. +Revision 55 supersedes D052's initial direct-formatter assumption. + +Adversarial review found that public two-argument raw constructors and low-level +`setScale` mutators can expose values outside the narrow classes' documented +precision and scale domains. Calling `Decimal128.toSink` directly silently +changed the full physical value `2^127 - 1` into 38 nines, while the server first +promotes it and emits all 39 digits. Adding a new local rejection policy would +also change previously accepted old-server writes and is unnecessary. + +The corrected binding lazily reuses one Decimal256 scratch, copies every native +decimal into it, applies the unsigned low QWP scale byte, then formats into the +existing reused text sink. This is the server's scalar algorithm. It reuses its +scratch and sink; after the sink reaches the required high-water capacity, +formatting needs no per-value object allocation. Public null sentinels remain +no-ops. + +Parity is deliberately per setter value, not per legacy column block. A known +STRING/VARCHAR target accepts mixed native widths and independently preserves +each value's effective scale. Legacy inference must choose one native wire width +and scale for the whole block, so it rejects a later width and may rescale or +reject a later scale. Recreating that intermediate constraint after the target +is known to be text would add buffer state and make a value's conversion depend +on neighboring rows; revision 55 rejects that complexity. Tests must name and +cover this behavior difference directly rather than calling it whole-batch +legacy parity. + +D052's proposed separate component fixture is also superseded. The public +socket test already drives the binding and asserts the exact bitmap, offsets, +bytes and end-of-frame boundary. Repeating that path through a lower-level +fixture would add test machinery without another observable contract. + +The first candidate JAR with direct narrow formatting and SHA-256 +`4af31d76224a3e811527114638240eabfe5939445f218c798c7f2fc3d1247b9e` +is rejected and retained only as superseded evidence. The accepted candidate +and installed JAR is +`e06759605ee3d1cf01c3277ba13531b0172e42138042c1f2f49c2db49e04b22d`; +only `QwpSchemaBinding.class` differs from the frozen phase-5d JAR. + +The stored production/server differential passes 3,000,294 deterministic +comparisons across full physical limbs, arbitrary integer scales, explicit +scale boundaries and all three public null sentinels. The 24-row shared corpus +is byte-identical in both repositories. Client validation passes the 46-test +public integration class, including four phase-5e tests, the 950-test canonical +gate, 1,275-test broad gate and two packaged-JAR tests. All 80 selected broad +XML reports have zero failures, errors and skips. + +Real-server validation passes all three Phase-5e E2Es, including mixed widths/ +scales, the full physical Decimal128 value and wrapped scale 257. The unchanged +original conversion diagnostic passes all 56 methods, resolving the original +47 failures. The accumulated server gate passes 363 tests. Three seeded repeats +pass 126 tests each; all 24 selected reports per run contain the requested seeds +and project-local temporary path with zero failures, errors and skips. No server +production or SQL-engine change belongs to phase 5e. + +The clean performance window contains six counterbalanced pairs for each native +identity width and 200 million setter calls per process. Instructions change by ++1.05% for Decimal64, +1.38% for Decimal128 and +0.71% for Decimal256; paired +median cycles change by -1.93%, +1.13% and -1.03%. Exact work and checksums +match, counters run at 100%, and no run has GC, major faults, throttling or OOM +changes. The result accepts the change against the 5% regression threshold but +makes no speed, decimal-to-text, network or ingestion-throughput claim. One +Maven-overlapped matrix and the withdrawn-candidate matrix are isolated and +excluded. Exact commands, hashes and logs are in the client and server 2.36 run +manifests. No commit or push is part of this checkpoint. + +Final independent SOL acceptance found no P0, P1 or P2 blocker after checking +production ownership, compatibility wording, exact hashes, raw logs, XML +provenance and performance claim bounds. One P3 remains: older source-map line +numbers in the design have drifted, although their method names remain correct. +That navigation debt is outside phase 5e and does not change the contract. diff --git a/design/schema-aware-sender-remaining-failures-plan.md b/design/schema-aware-sender-remaining-failures-plan.md new file mode 100644 index 000000000..f38761324 --- /dev/null +++ b/design/schema-aware-sender-remaining-failures-plan.md @@ -0,0 +1,693 @@ +# Plan: resolve the remaining 47 schema-aware Sender test failures + +Status: all five phases accepted through iteration 2.36, 2026-09-12. The user +requested sequential completion of the remaining conversions. All original 47 +failures now pass. +Follow the [main design](schema-aware-sender.md); +decision history and evidence belong in the [journal](schema-aware-sender-journal.md). + +## Recommendation and scope + +Fix the nine stale expectations/fixtures first, then add conversions in small +end-to-end increments: text output, timestamps, small scalar targets, decimals. +Do not treat 47 failing methods as 47 production bugs, or implement a generic +conversion framework to make them disappear. + +All functional changes are in QWP client encoding/binding. No SQL parser, +planner or execution-engine change is proposed. SQL in these tests creates +tables and verifies stored results. + +| Phase | Baseline failures addressed | Count | Baseline failures left after the phase | +| --- | --- | ---: | ---: | +| 1 | Existing semantics and invalid test fixtures | 9 | 38 | +| 2 | LONG, UUID and floating-point text output | 8 | 30 | +| 3 | Ordinary timestamp input/output conversions | 4 | 26 | +| 4 | CHAR, LONG256 and GEOHASH targets | 4 | 22 | +| 5 | Decimal conversions | 22 | 0 | + +These are coverage checkpoints, not instructions to force the counts down. +A method is resolved only when its intended contract is tested and passes. +The substeps below are separate reviewable increments, not one large phase PR. + +Current checkpoint: phases 1–4 and phases 5a–5e are accepted. Phase 5a covers +all 18 logical native-decimal pairs; phase 5b covers LONG into all six decimal +target classes; phase 5c covers STRING into all six; phase 5d covers FLOAT and +DOUBLE into all six; phase 5e covers all native decimal widths into STRING and +VARCHAR. The original 56-test conversion class is green. Exact-current client +validation passes the 46-test public integration class, including four phase-5e +tests, plus two packaging checks, 950 canonical tests and an expanded 1,275-test +regression gate. Server validation passes three Phase-5e E2Es, all 56 original +conversion tests, 363 canonical tests +and three 126-test seeded repeats with 24 selected XML reports each. +Legacy mode, row recovery and pinned encodings are preserved. +D041 records the NaN-null approval and formatter work; D042 records the small +raw timestamp extension, corrected wire fixtures and native LONG measurements. +D043 records the fixed timestamp parser, its source-quirk regressions, broad +matched-runtime differential and the measured 2–3% native string-write cost. +D044 records checked timestamp-to-text normalization, exact server formatting, +63 shared vectors and 160,202 public differential checks. Its native timestamp +performance guard detects no repeatable regression. +D045 records CHAR's first-character conversion, exact wire-presence versus +SQL-null distinction, 23 shared vectors, 65,549 public differential checks and +the missing standard schema-refresh catch caught by the new setter's own test. +D046 records STRING-to-LONG256's four-local-limb parser, 23 shared vectors, +40,055 public differential checks, exact rollback/generation wire tests and +independent legacy acceptance/rejection. The native string-write regression +guard passes; it is not a LONG256 throughput measurement. +D047 records all 60 GEOHASH precisions, exact schema bitmap framing, empty/null +normalization, 7,210 public differential checks and canonical packed-type +validation. The two-path performance guard passes with measured small costs: +about 1.46–1.80% append time and 0.17–1.15% encoding time in native string checks. +D048 records phase 5a's exact rescale/precision/storage checks, target-scale +retention, caller immutability, MISSING/legacy behavior, row rollback/schema +generation and bitmap-only decimal null characterization. Its 60-row shared +corpus has 45 values and 15 invalid cases across all 18 logical pairs (SHA-256 +`3819694736f54de4d2f2f396b09ff487667051c6045b34cc6d7579552551d11d`). +The frozen candidate JAR SHA-256 is +`1e814a2f82836b5c9ac04cf7b5674e56f1b939440df486ae6628ebb1b8eebb08`. +The three seeded 101-test repeats have 20 archived XML reports each. +D049 records phase 5b's scale-zero LONG conversion, stable schema-mode +`Long.MIN_VALUE` null, target-scale locking, direct target-width wire output, +rollback and schema-generation coverage. Its shared corpus has 39 rows: 19 +values, 14 invalid values and six null cases across every decimal target class +(SHA-256 +`d9a47432a14583f8675b0035ab10c3ef9f36e78c9f4aab6a8634958eb5aafcb1`). +The frozen candidate JAR SHA-256 is +`41d137a1c2388ed21f776e372771785377d91c15a887d94f04e82c97f505d403`. +The 12-run native LONG-to-LONG guard detects no repeatable 5% cycles or +instructions regression; it is not decimal-throughput evidence. +D050 records phase 5c's target-precision/scale parser, direct target wire output, +stable bitmap-null rules and deliberate correction of three unsafe server +behaviors. Its 55-row shared corpus has 19 values, 24 invalid values and 12 +nulls (SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`). +The installed candidate JAR SHA-256 is +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +The 3,601,944-case differential passes ordinary server parity and records 72 +deliberate malformed-prefix corrections. The 12-run native STRING-to-VARCHAR +guard detects no repeatable 5% cycles or instructions regression; it is not +decimal, network or ingestion throughput evidence. +D051 records phase 5d's exact FLOAT widening, shortest round-tripping decimal, +lossless target-scale check, non-finite bitmap nulls and direct target-width +output. Its shared corpus has 60 rows: 37 values, 17 invalid values and six nulls +(SHA-256 +`366a889719c6e7b688ff00e1bdfb95e273ab192508a1094dac5312c3d899611d`). +The 5,996,868-case differential matches the server's acceptance, target scale and +limbs. The installed candidate JAR SHA-256 is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +Performance validation caught and removed a native hot-inlining regression by +keeping decimal conversion in the existing cold nonnumeric helper. The final +12-run guards detect no material native FLOAT/DOUBLE regression; they do not +measure decimal conversion, network or ingestion throughput. + +D053 records phase 5e's corrected Decimal256 promotion, unsigned scale-byte +semantics, full-physical public values and intentional per-value mixed-width/ +scale behavior for known text targets. Its 24-row shared corpus has 18 values +and six public null cases across all six source/target pairs (SHA-256 +`4fd43e6d1c08fc76cee6505e6ef06a250aa30281bd686bb0115ba6c710065b5f`). +The 3,000,294-case differential matches server scalar formatting over full +physical limbs and arbitrary integer scales. The accepted and installed JAR is +`e06759605ee3d1cf01c3277ba13531b0172e42138042c1f2f49c2db49e04b22d`. +Six counterbalanced pairs for each native identity width show instruction +changes of +1.05%, +1.38% and +0.71%, with no repeatable 5% regression. This is +an identity-path guard, not a decimal-to-text, network or ingestion benchmark. + +Baseline: iteration 2.20, client HEAD +`981bdb02a471f3b290c89b8e78cbc422610e329e`, server HEAD +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with the recorded dirty working +trees. Frozen client JAR SHA-256: +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +Server evidence is in +`core/target/schema-text-source-null.2.20-server/run-manifest.md`, +`logs/broad.log` and `logs/broad-failure-names.txt`: the fixed 333-test +selection has seven failures and 40 errors, exactly 47 methods. The separate +271-test green gate and three 26-test seeded repeats passed. The inventory at +the end assigns every failing method exactly once. + +## Phase 1: restore honest tests — 9 failures + +### 1a. Separate existing schema and legacy behavior — 5 + +Accepted in iteration 2.21. The following describes the completed repair. + +The three boolean tests already exercise working client conversions but expect +legacy omissions to become false/zero. Update the schema expectations to the +target's missing value: null for nullable targets, zero for BYTE/SHORT. Assert +nullness as well as printed values. Keep the existing independent unflagged +legacy comparison in `QwpSchemaBooleanE2ETest`. + +The two floating tests deliberately describe legacy behavior: supplied NaN can +depend on the bitmap, and positive 2^63 can clamp to LONG maximum. Their current +public Sender calls instead apply the already-approved schema rules. Send the +legacy cases as actual unflagged FLOAT/DOUBLE frames, preserving both the +accepted values and server rejection checks. Keep explicit schema tests for +NaN normalization and local rejection at 2^63 in +`QwpSchemaFloatingNumericE2ETest`; do not restore legacy behavior in production. + +### 1b. Remove direct mutation of Sender-owned buffers — 4 + +Accepted in iteration 2.22. The following describes the completed repair. + +The two geohash-to-text and two designated-LONG timestamp tests obtain a buffer +through `getTableBuffer()` and manually add source columns. Subsequent public +`at()`/`atNow()` installs schema bindings and uses extended framing, so these +are no longer valid fixtures. + +Rebuild them with a standalone native buffer, ordinary unflagged encoder and +nonrequesting WebSocket client. Assert source types, geohash precision, bitmap, +values, ACK and exact stored rows. Keep named LONG designated-timestamp wire +semantics separate from public `at()`/unit conversion. Do not relax Sender +ownership or add a schema opt-out. + +The public native geohash converter remains unimplemented after this repair. +Keep that fact visible; a passing legacy wire test does not implement it. + +## Phase 2: text output — 8 failures, three increments + +### 2a. LONG to STRING/VARCHAR/SYMBOL — 3 + +Accepted in iteration 2.23. The following describes the implemented contract. + +Approved on 2026-09-11 and recorded in the main contract before code: +supplied `Long.MIN_VALUE` becomes a target null, as an effective write rather +than a no-op. Ordinary values use canonical signed decimal text. The legacy +bitmap-present special value is the four-character text `"null"`, not decimal +minimum; preserve the 2.20 raw characterization. + +Use the existing target VARCHAR representation for STRING/VARCHAR and existing +dictionary-backed SYMBOL representation for SYMBOL. Reuse verified formatters +and append paths, with no raw-LONG fallback. Cover MIN+1/MAX, zero, negatives, +omission, explicit sentinel, repeated symbols and duplicate-first behavior. + +### 2b. UUID to STRING/VARCHAR — 2 + +Accepted in iteration 2.24, following separate approval on 2026-09-11. Only the pair +with both limbs equal to MIN becomes target SQL NULL, as an effective write. +A single minimum limb remains a real value. Preserve `(lo, hi)` input order and +canonical lowercase formatting. Do not add UUID-to-SYMBOL or change native UUID +bytes, inference or legacy behavior. The verified checkpoint is 35 to 33, with +only the two UUID-to-text baseline failures removed and no additions. + +The obsolete public UUID-to-text rejection cases in +`QwpSchemaTextSourceNullE2ETest` now exercise unsupported UUID-to-SYMBOL instead; +its independent legacy-byte tests are unchanged. Shared literal vectors, +null-first ignored duplicates, omission/reset, public row recovery, native/text +encoding generations and component-seeded text replay are covered. + +### 2c. FLOAT/DOUBLE to STRING/VARCHAR/SYMBOL — 3 baseline methods + +Accepted in iteration 2.25. The following describes the implemented contract. + +Cover the floating input family coherently, including the FLOAT overload +although these three failing methods happen to use DOUBLE. Match the server's +actual formatter, including FLOAT widening before formatting. Check signed zero, +finite rounding boundaries, infinities and NaN with/without an omitted row. + +Separately approved on 2026-09-11: supplied FLOAT/DOUBLE NaN becomes SQL NULL +for all three text/SYMBOL targets, as an effective first write. Legacy raw +characterization confirms bitmap-present NaN becomes literal `NaN` instead. +Do not substitute Java formatting or add a per-row policy switch. Differential +tests found that the older client formatter differs from current server Ryu; +keep source-faithful formatting isolated from legacy paths. + +## Phase 3: ordinary timestamp conversions — 4 failures + +### 3a. LONG to timestamp — 1 + +Accepted in iteration 2.26. The following describes the implemented contract. + +Reuse target timestamp encoding. LONG denotes a raw count in the target unit, +not a value whose unit is inferred or converted from micros. Test both micro- +and nanosecond targets, negative/zero/boundary values and source-null behavior. +This must not enable named writes to the designated timestamp. + +### 3b. STRING to timestamp — 1 + +Accepted in iteration 2.27. The following describes the implemented contract. + +Port/reuse the server's actual timestamp grammar and value checks, not an SQL +cast or an assumed ISO-only parser. The server uses its microsecond parser +and checked multiplication by 1000 for a nano target. Test accepted/rejected +grammar, overflow, fractional precision, null/omission and failed-row recovery. + +Use a fixed parser for the eleven actual formats, preserving their priority, +not the server's general compiler. Copy the relevant calendar helpers exactly: +`yearMicros` has asymmetric negative-overflow saturation, while surrounding +arithmetic and timezone subtraction can wrap. The narrow zone adapter must +retain locale-token ordering and use the separately parsed year for future +rules, even after epoch arithmetic wraps. Plain JDK instant lookup differs on +a reproduced extreme-year input (D043). Use public JDK metadata, not reflection +or a general timezone engine. Same-runtime differential tests must precede +acceptance; runtime locale/tzdb data and the class-initialized reference century +remain independently sourced by client and server. Do not quietly narrow the +grammar or add protocol metadata to this increment. + +### 3c. Timestamp to STRING/VARCHAR — 2 + +Accepted in iteration 2.28. The following describes the tested contract. + +Cover both primitive timestamp units and `Instant`, not only the MICROS calls +in the failing tests. The extra overloads are coherent-family acceptance +coverage, not additional failures in the 47-method count. +The server formats text to millisecond precision; nanoseconds first +divide by 1000 with truncation toward zero to obtain micros, then +`MicrosFormatUtils.appendDateTime` uses the millisecond UTC pattern. Characterize +negative sub-microsecond values, extremes and source-null behavior. Existing +timestamp-target range/precision exceptions do not silently extend to text. +The timestamp cursor uses bitmap nulls only: present MICRO MIN formats as empty +text, while present NANO MIN formats as a finite historical timestamp. Preserve +these semantics; the LONG/NaN text-null approvals do not apply by analogy. +Separately approved by the user: checked conversion into the legacy source +precision for larger units and Instant, rejecting out-of-range values before +append. Keep legacy mode unchanged. D044 records this text-target overflow +decision; it is not an implicit extension of D021. + +## Phase 4: small scalar targets — 4 failures + +### 4a. CHAR identity and STRING to CHAR — 2 + +Accepted in iteration 2.29. The following describes the completed slice. + +Use target-native CHAR encoding and the existing native CHAR setter. For text +input, reproduce QWP's first decodable BMP-character behavior. Empty input, +malformed raw UTF-8 and a supplementary first code point produce NUL. Test +public malformed UTF-16 separately: `stringColumn` applies replacement during +UTF-8 encoding, so assert the actual replacement bytes and resulting CHAR, +not an assumed NUL. Test more than the three ASCII values in the old test. + +Do not bundle CHAR-to-text: its non-ASCII VARCHAR behavior is a separate, +documented server compatibility issue. + +### 4b. STRING to LONG256 — 1 + +Accepted in iteration 2.30. The following describes the completed slice. + +The target LONG256 representation and server-equivalent parser/limb order use +existing value storage. Tests cover accepted syntax, invalid/overflow input, +null, leading zeros, boundaries, exact wire bytes and failed-row recovery. + +Read-only source verification after 2.29 confirms a lowercase `0x` prefix and +2–64 hex digits in complete byte pairs; hex digits themselves are case-insensitive. +Reject signs, whitespace, odd digit counts and overlong input. Parse right-to-left +into four local longs and append with existing addLong256; no retained scratch +object is needed. Java null/omission is null, but the literal whose four limbs +are all Long.MIN_VALUE must be rejected, matching the server parser/appender. + +Native LONG256 setter activation remains separate, unimplemented work. + +### 4c. STRING to GEOHASH — 1 + +Accepted in iteration 2.31. The following describes the completed slice. + +Bind target bits from the schema and encode the parsed value with that precision. +Match the server's base32 parsing, truncation, empty/null and consumed-character +rules, including its 12-character cap. Test multiple precisions and invalid or +insufficient input, not just GEOHASH(4c). + +Use the precision already packed in the server's column type. The current +schema response sends that full type code and an empty extension block; +decimal precision/scale is likewise already encoded in its type code. Do not +add new protocol parameter fields or stop rejecting unknown extension blocks +just to enable these types. Keep precision pinned with the existing schema +snapshot. Native geohash input/text-output activation remains separately tracked; +the phase-1 raw fixture repair does not supply it. + +Read-only preparation after 2.30 identifies one storage detail to address: +`addGeoHash` currently sets precision only on a value write, but a null-only +column must also carry the schema's precision. Reuse the existing precision +field and keep the adjustment narrow. + +Source revalidation for 2.31 corrects the initial empty-value proposal. Native +GEOHASH without a bitmap treats a packed all-ones value as null, losing valid +maximum values at byte-aligned precisions. With a bitmap, packing empty text +as all ones can instead produce a non-null value when packed width is smaller +than storage width. Preserve the server's VARCHAR conversion result: normalize +empty text to bitmap null, and always include the bitmap for schema-directed +GEOHASH columns, including all-present batches. This costs one bit per row and +needs no value scan or new retained state. Force only columns already using +nullable storage; low-level non-nullable schema buffers and legacy encoding +remain unchanged. Runtime characterization and exact wire tests establish both +cases: source-wire presence does not imply target-wire presence. Recognize only +the complete canonical packed type for a valid precision, preserving rejection +of malformed and future-flagged types. + +## Phase 5: decimals — 22 failures, five increments + +Keep decimals last because target precision/scale, source scale, wire width and +null semantics are shared dependencies. Reuse existing decimal arithmetic and +8/16/32-byte column storage only after checking their behavior against the server. + +| Increment | Baseline methods | Required behavior | +| --- | ---: | --- | +| 5a. Native decimal to decimal (accepted in 2.32) | 4 | Target precision/scale and wire representation; all source widths; exact rescale, narrowing and overflow | +| 5b. LONG to decimal (accepted in 2.33) | 4 | Checked scaling and target precision/storage limits; explicit source-null rule | +| 5c. STRING to decimal (accepted in 2.34) | 4 | Server-equivalent grammar and target-scale parsing; precision-loss/overflow failures | +| 5d. FLOAT/DOUBLE to decimal (accepted in 2.35) | 4 | Actual server rounding/conversion behavior, non-finite values, underflow and overflow | +| 5e. Native decimal to text (accepted in 2.36) | 6 | All three input widths to STRING/VARCHAR, preserving per-value scale and server scalar formatting | + +5a is accepted in iteration 2.32 and produces a working real-Sender E2E before +the later input families are added. Small decimal targets use DECIMAL64 wire +storage while enforcing the narrower target precision and scale. Convert +every input width/scale to the target precision, scale and compatible storage +width at setter time. The encoded decimal scale must therefore equal the +target scale; the original input scale is only conversion input. Assert that +wire scale and exact converted limbs in cross-width E2Es. Do not merely relabel +source limbs or leave rescaling/precision validation to the server. Reuse +existing append storage by supplying already-converted values, rather than +adopting its legacy first-input-scale rule for schema bindings. +Cover emitted all-null columns separately from null setters that omit the +column entirely; neither has a non-null coefficient to rescale. + +Read-only preparation after phase 4c finds an existing per-column Decimal256 +scratch value. Reuse it for copy/rescale/validation before append rather than +adding another binding-owned scratch object. Preserve caller-owned input values. +Keep internal high-to-low decimal words separate from the writer's low-to-high +wire order. Also preserve the schema's target scale across reset and failed-row +cleanup: a following batch may emit that retained column entirely as nulls, +without invoking a decimal setter. Legacy columns must keep their current +first-input-scale-per-batch behavior. Prove both paths before choosing the +smallest metadata-lifetime change; do not share a new blanket reset policy. + +The accepted implementation covers all 18 logical pairs with nine physical +source-width/target-width paths. It copies each caller value into the existing +per-column Decimal256 scratch, performs exact rescale plus target-precision and +storage-width checks, then appends the direct target wire width. Target scale +survives reset and failed-row cleanup. Caller objects are unchanged; a failed +partial row rolls back without damaging completed rows or schema-generation +boundaries. Confirmed MISSING and legacy paths retain source-width inference, +public null/no-op behavior and the legacy first-input-scale rule. + +Decimal cursors use bitmap-only nullness, not the fixed-width sentinel rule. +Characterize raw sentinel limbs with and without a bitmap and preserve public +typed-decimal null/no-op behavior. Check representable source scales and +representative malformed scale bytes; if an existing server validation gap is +found, report it separately rather than copying unsafe acceptance into the +client or quietly broadening this plan into a server rewrite. + +That characterization is complete for 5a: explicit bitmap nulls are null, while +raw sentinel limbs without a null bitmap remain supplied conversion input at the +server boundary. At the 2.32 checkpoint this did not activate decimal-to-text +or the later input families. + +5b is accepted in iteration 2.33. For a known decimal target, LONG starts as an +exact scale-zero integer in the existing per-column Decimal256 scratch, then +uses 5a's rescale, precision/storage validation and target-width append tail. +No conversion registry, second scratch value or Sender/protocol state was added. +Small decimal targets still use DECIMAL64 wire storage while enforcing their +declared precision. MISSING still infers LONG. + +The schema-mode source-null rule is now explicit: `Long.MIN_VALUE` becomes a +decimal bitmap null and locks the target scale before append. Its meaning is +therefore independent of other rows' omissions. Legacy encoding and server +conversion are unchanged and retain the existing bitmap-dependent result, +which a raw-frame real-server test characterizes separately. Duplicate +suppression happens before validation, and an invalid partial row is rolled +back while completed rows remain. A schema-generation test proves that a +sealed DECIMAL64(scale 4) block retains its bytes when a refresh selects +DECIMAL128(scale 5) for the later block. + +The 39-row shared client/server corpus covers all six target classes and has +19 accepted values, 14 local rejections and six source-null cases. Component +tests assert exact type, bitmap, scale and low-to-high limbs; public Sender E2Es +assert exact stored values and recovery. The original four server conversion +methods fail against the frozen 5a client and pass against 5b. After acceptance, +the unchanged original diagnostic has 14 errors, all assigned to 5c–5e. + +The legacy-path performance guard passes 12 balanced A/B runs: instructions +change by +0.001% to +0.002% and paired cycles by -0.47% to +1.99%. A separate +candidate-only nine-physical-pair characterization has six runs of 20 million +rows each (180 million conversions and 3.36 GB verified per run), with basic +medians of 32.91 ns, 182.03 cycles and 1140.18 instructions per value. It has no +matched schema baseline and makes no network, ingestion or general-throughput +claim. One 200-million-row sizing run and an interrupted follow-on are excluded. + +5c is accepted in iteration 2.34. For a known decimal target, STRING is parsed +at the target precision and scale with `strict=false` and `lossy=false`, including +the server grammar's exponent underscores. The existing Decimal256 scratch then +goes through phase 5a's precision/storage checks and direct target-width append +tail. The shared parser uses the current server's leading-zero and full-scale +precision rule; a schema-only parser fork was not added. Small decimal targets +continue to use DECIMAL64 wire storage while enforcing their declared precision +and scale. + +Java null and exact optionally signed `NaN` or `Infinity`, after the parser's +supported suffix processing, become target-scale bitmap SQL nulls for all six +target classes. Malformed special-value prefixes and extreme nonzero integer- +boundary exponents fail locally as `INVALID_VALUE`. Schema mode also avoids the +server's small-decimal special-value narrowing bug by emitting bitmap nulls. +The exact-special and exponent checks live in the shared parser, so direct +`Decimal64`, `Decimal128` and `Decimal256` string parsing changes globally: it +rejects malformed special prefixes and reports extreme nonzero exponents as +`NumericException`. MISSING and legacy Sender paths remain VARCHAR and retain +server-side conversion. + +The implementation adds no state, framework, protocol, server production, SQL +or send-time transformation. The 55-row shared corpus contains 19 values, 24 +invalid inputs and 12 nulls and has SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`. +Client red validation failed eight tests before implementation. Final focused +validation passes eight tests plus two packaging checks, all-decimal validation +passes 463 tests, and the canonical gate passes 941. The candidate and installed +JAR SHA-256 is +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +The 3,601,944-case differential passes ordinary server parity and identifies 72 +intentional malformed-prefix corrections. Synchronizing shared-parser precision +metadata corrected 19 stale assertions in 16 existing tests without changing +their parsed values or scales. + +Server focused validation passes seven tests and the canonical gate passes 352. +The original 56-test conversion class has ten errors: only the four +STRING-to-decimal methods disappeared, with no additions. Three seeded repeats pass 115 tests and +22 archived XML reports each. The 12 balanced fixed-work native +STRING-to-VARCHAR runs share checksum `15640344951909530213`; paired basic +instructions change by -0.001% to -0.000% and cycles by -1.028% to -0.007%, +with no repeatable 5% regression. This makes no decimal, network or ingestion +throughput claim. + +5d is accepted in iteration 2.35. FLOAT is widened exactly to DOUBLE, the existing +schema Ryu formatter emits the shortest round-tripping decimal, and the existing +decimal parser applies the target precision and scale with `lossy=false`. +Natural-scale and precision overflow therefore reject instead of rounding or +truncating. NaN and both infinities become target-scale bitmap nulls, matching all +six server decimal paths. This is intentionally not the SQL cast contract. + +The implementation reuses the phase-5a Decimal256 scratch, validation and direct +width append tail. It adds no new parser, state or conversion framework. The +first branch placement increased native FLOAT/DOUBLE instructions by about +3.8–4.7% after pushing a common method over the hot inlining threshold. Moving +decimal dispatch into the existing cold nonnumeric helper restores the common +method's prior bytecode shape. Six counterbalanced pairs per native path then +show median instruction changes of +0.0001% for FLOAT and +0.0037% for DOUBLE. + +The 60-row shared corpus covers every source/target-class pair, exact width, +scale and limbs, no-rounding cases, non-finite nulls, signed zero, underflow and +overflow. Client and public Sender tests cover duplicate-first-wins, omission, +invalid-row rollback and a DECIMAL64-to-DECIMAL128 schema generation change. +Four real-server E2Es consume the same corpus and independently preserve raw +unflagged legacy FLOAT/DOUBLE bytes. The unchanged 56-test diagnostic now leaves +only the six 5e errors. + +5e is deliberately binding-only. For a known STRING or VARCHAR target, promote +the supplied value into one lazily reused binding-owned Decimal256, reduce its +scale to the unsigned low QWP byte, format that promoted value into the existing +reused text sink, and append `TYPE_VARCHAR`. This mirrors the server's scalar +conversion and avoids the narrow formatters' precision limit for full physical +Decimal64/128 values. Preserve valid source scales, trailing zeros and plain +notation. Public Java null and decimal `NULL_VALUE` remain setter no-ops; they +do not claim first-value-wins. MISSING and legacy mode keep native decimal tags +and the original source scale. Do not add a formatter abstraction, conversion +registry or table-buffer state. + +Scalar parity is not legacy batch emulation. Once schema binding establishes a +text target, every setter value is converted independently, so mixed native +decimal widths and scales in one target column are supported. Legacy inference +locks one native width and scale for a column block and may reject or rescale a +later value. Recreating that irrelevant intermediate constraint would add state +and make text conversion depend on neighboring rows, contrary to the schema- +directed design. Cover this intentional behavior difference explicitly. + +The shared corpus must cover all six source/target pairs, each physical width, +positive, negative and zero coefficients, maximum precision/scale, leading and +trailing fractional zeros, and both public null forms. Public socket tests assert +the exact `TYPE_VARCHAR` bitmap, offsets and bytes while covering null-then-value, +duplicate-first-wins before conversion, A/failed-B/C rollback, omission, scratch +reuse and a DECIMAL64-to-VARCHAR schema-generation change. A second component +test would repeat the same binding path without adding an observable contract. +Real-server tests run the same valid corpus and an independent unflagged native- +decimal frame so the old server formatter remains covered. + +Public low-level constructors and `setScale` mutators can expose coefficients +or scales outside the documented 18/38/76-digit domains. Phase 5e does not add +a local rejection policy: Decimal256 promotion preserves the physical value and +the unsigned low scale byte matches what the server receives. Public sentinel +values remain setter no-ops, so non-bitmap sentinel formatting stays confined +to independent raw-frame characterization. Any Decimal API or server +rejection/normalization is separate work. + +## Acceptance required for every increment + +1. Read the entire affected test, its public setter path, target dispatch and + concrete parser/formatter overloads. Confirm the failure from the frozen + baseline; save pre-edit copies of every file to be changed. Record any + deliberate compatibility exception in the main design before code. +2. Implement only the selected family in the existing binding/buffer ownership. + No second schema cache, conversion registry, new connection state, opt-out, + send-time transformation, or source-type fallback. Avoid unrelated guards + and hot-path refactors; check shared-path allocation/throughput regressions + with comparable fixed work when those paths change. Do not claim a speedup + from source inspection or test timings. +3. Add a public-wire test proving direct target types and parameters. Add a + separate component test only when it covers a distinct observable contract. + Use shared, identical conversion vectors for client/server tests where + applicable. A server SQL result alone cannot prove client conversion. +4. Keep a real public-Sender E2E passing in that same increment. Cover completed + row A, partial invalid row B, valid row C; automatic rollback without + cancellation/reselection; first-value-wins; null vs omission; boundary values; + typed local errors; schema change/rebinding and replay where affected. + Pin auto-flush when asserting buffered-row preservation and await the + published ACK when assertions depend on feedback. +5. Preserve independent legacy native-wire acceptance/rejection tests and + byte-identical replay. A test helper must not accept either a local or a + server error interchangeably. SQL is only setup/result verification. +6. Run client component/public-Sender regressions for production changes, then + build/install one standalone client artifact. Freeze its hash before server + validation; no overlapping client install and server tests, no + `-Plocal-client`. Use an absolute project-local temp directory in the + Surefire JVM startup `argLine`, not merely a later system property. +7. Extend the current 271-test green server gate with the increment's tests and + recovered baseline cases. Rerun the unchanged 333-test diagnostic and compare + exact failing methods, not only totals. Run three seeded repeats of affected + integration/recovery cases; preserve failing development logs. +8. Independent SOL source/adversarial review plus architect acceptance. Update + this ledger, the main conversion inventory and the journal with source/JAR + hashes, commands, actual results and any next blocker. + +If a resolved first error reveals a later stale assertion, classify that +assertion against the approved contract. Do not stop at the first green setter, +delete the test, loosen its expected error, or silently count a renamed/removed +method as fixed. Readiness requires every baseline case's behavior to remain +represented. + +## Definition of done and remaining release work + +Done for this plan: all 47 baseline cases are represented and pass, the original +333 selection is green (or has an explicitly recorded one-to-one test rename), +the accumulated green gate/client regressions pass, and independent legacy +checks retain their intended behavior. No unresolved new regression is accepted. + +This is not the complete feature-release checklist. The design still inventories +unrepresented BYTE/SHORT/INT families, DATE targets, IPv4, arrays, native geohash/ +LONG256 paths and other pairs, plus compatibility decisions for unsafe server +paths. Actual older-server-distribution testing, lookup-timeout configuration, +full lifecycle/compatibility validation and representative performance evidence +also remain release work. Do not advertise full schema conversion coverage just +because these 47 methods turn green. + +## Exact 47-method ledger + +Source files: +[conversion E2Es](../../questdb/core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpWebSocketTypeConversionE2ETest.java), +[Sender E2Es](../../questdb/core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpSenderE2ETest.java). +Group IDs match the increments above. All 47 entries are accepted. No case has +been removed or renamed in this original selection. + +### 1a. Existing schema/legacy expectations — 5 accepted + +- `QwpWebSocketTypeConversionE2ETest.testBooleanToNumericColumns` +- `QwpWebSocketTypeConversionE2ETest.testBooleanToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testBooleanToVarcharColumn` +- `QwpSenderE2ETest.testFloatingPositiveTwoToThe63ClampsToLongMax` +- `QwpSenderE2ETest.testFloatingToIntegerNaNDependsOnBlockNullBitmap` + +### 1b. Direct-buffer legacy fixtures — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testGeoHashToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testGeoHashToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToDesignatedTimestampMicroColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToDesignatedTimestampNanoColumn` + +### 2a. LONG to text/SYMBOL — 3 accepted + +- `QwpWebSocketTypeConversionE2ETest.testLongToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToSymbolColumn` + +### 2b. UUID to text — 2 accepted + +- `QwpWebSocketTypeConversionE2ETest.testUuidToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testUuidToVarcharColumn` + +### 2c. Floating point to text/SYMBOL — 3 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDoubleToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToSymbolColumn` + +### 3a. LONG to ordinary timestamps — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testLongToTimestampColumn` + +### 3b. STRING to ordinary timestamps — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToTimestampColumn` + +### 3c. Timestamp to text — 2 accepted + +- `QwpWebSocketTypeConversionE2ETest.testTimestampToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testTimestampToVarcharColumn` + +### 4a. CHAR target — 2 accepted + +- `QwpWebSocketTypeConversionE2ETest.testCharToCharColumn` +- `QwpWebSocketTypeConversionE2ETest.testStringToCharColumn` + +### 4b. STRING to LONG256 — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToLong256Column` + +### 4c. STRING to GEOHASH — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToGeoHashColumn` + +### 5a. Native decimal to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDecimalToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToSmallDecimalColumn` + +Residual P3: a scale-up overflow has the correct `INVALID_VALUE` reason and row +rollback, but its detail currently says `decimal value cannot be rescaled +exactly` instead of identifying overflow. Keep this as diagnostic wording debt; +do not add another hot-path branch solely to refine the message. + +### 5b. LONG to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testLongToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testLongToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testLongToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testLongToSmallDecimalColumn` + +### 5c. STRING to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testStringToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testStringToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testStringToSmallDecimalColumn` + +### 5d. Floating point to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDoubleToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToSmallDecimalColumn` + +### 5e. Native decimal to text — 6 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDecimalToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal128ToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal128ToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal256ToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal256ToVarcharColumn` diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md new file mode 100644 index 000000000..f65c881e4 --- /dev/null +++ b/design/schema-aware-sender.md @@ -0,0 +1,1703 @@ +# Schema-aware sender: schema-directed encoding + +Status: proposal, revision 55. Scope: QWP v1 over WebSocket, with an automatically +negotiated schema extension, legacy-server compatibility and companion server +changes. + +## Goal and contract + +On supporting servers, convert values to the server column's type before +buffering them. Conversion failures are reported by the setter and cancel only +the partial row. Completed rows stay buffered. Against +old servers, retain the existing inferred-type behavior and server-side +conversions; those writes do not gain early schema-mismatch detection. + +For a server UUID column, `stringColumn("id", validUuid)` parses locally and +appends a binary UUID; `stringColumn("id", "invalid")` throws +`LineSenderException`. `uuidColumn("id", lo, hi)` writes directly to the same +UUID buffer. The setter describes the input type; the server schema determines +the buffered representation. + +This replaces revision 4's compatibility checks. Knowing that STRING-to-UUID +is supported cannot validate a particular string. The client now performs the +conversion, rather than merely checking an acceptance mask. + +Schema mode has three parts: + +1. Obtain the server schema synchronously on first use or a cache miss. +2. Bind columns to that schema and convert every effective value before append. +3. Pin each table block to its encoding snapshot. ACKs/NACKs provide newer + schemas for subsequent blocks, without changing already converted data. + +The guarantee is local conversion validity against the pinned schema, not +eventual server acceptance. Concurrent DDL, auto-creation races and other +server-side failures can still reject a batch. + +## Implementation status + +Current checkpoint: iteration 2.36 completes phase 5e native-decimal-to-text +conversion for all three native widths and both text targets. All original 47 +failing conversion cases now pass. +The wider conversion inventory below is not yet complete; this is not release +acceptance. + +Schema discovery, pinned identity framing, safe replay and ACK/NACK feedback +are implemented and tested. Local UUID and LONG-to-numeric conversion now uses +a non-owning binding over a caller-owned `QwpTableBuffer`; the caller retains +row completion, rollback and buffer lifetime. + +First-use lookup and forced refresh now run through the existing cursor I/O +loop and share a bounded latest-schema cache with ACK/NACK feedback. Tests +cover lookup failure/lifecycle handling and a real-server lookup, schema change, +ACK-driven cache update and same-buffer rebind, including fragmented responses. + +Primitive timestamp units from NANOS through DAYS and `Instant` conversion into +non-designated timestamp columns are implemented and tested against real-server +ingestion. Coverage includes range boundaries, target precision, nulls, rollback, +compressed encoding, recovered delivery and deliberate legacy differences. +STRING-to-timestamp parsing is accepted in iteration 2.27. Designated timestamp row completion is +included in the Sender integration described below. + +Text and symbol inputs into STRING, VARCHAR and SYMBOL targets are implemented +and tested with 90 shared cases, exact wire/stored-value checks, legacy value +parity and local-dictionary SF recovery. Client tests also cover Sender-owned +global symbol encoding. + +Boolean inputs and strict text-to-boolean parsing are implemented and tested +against real-server acceptance/rejection, target-native omissions, packed nulls +and recovered delivery. + +FLOAT/DOUBLE input conversion into the six numeric targets is implemented and +tested: NaN means missing and out-of-range integer conversions fail locally. +Coverage includes raw-bit rounding boundaries, legacy differences, rollback +and recovered delivery. D025/D026 record the decisions and verification. + +STRING input conversion into the six numeric targets is implemented and tested +with 429 shared cases, exact wire/stored values, legacy acceptance/rejection, +null/sentinel contexts, rollback and recovered delivery. Parsing preserves +target-specific syntax, including floating suffixes; overflowing LONG text +fails locally under the approved D027 compatibility exception. + +BINARY targets now accept all three binary input forms and STRING input encoded +as UTF-8, using native BINARY wire representation before persistence. Component and +real-server tests cover exact bytes, source ownership, null versus empty, +duplicate suppression, local errors and rollback. A shared variable-width +append repair also preserves completed rows if string encoding throws midway; +successful legacy writes retain their encoding. D029 records the accepted slice, +independent reviews and complete verification evidence. + +Iteration 2.15 is accepted as a bounded, unreleased Sender integration. +Public setters select the negotiated mode, use server-directed conversions, +cancel failed partial rows and retain completed rows under their original +schema. Both designated timestamp units and cold `atNow()` are integrated. +Eight real-server tests cover these operations, relevant/unrelated DDL, +schema lookups, auto-flush, reset and close. Fifteen public-Sender socket +tests separately cover exact wire encoding, legacy fallback, one-way upgrade, +startup timeout/interruption, pinned generations and split frames. +The broader server regression selection still has 38 classified failures +after the 2.22 expectation/fixture repairs; +this is not a release-ready client. Compatibility with a non-confirming socket +peer is tested; an actual older QuestDB distribution was not run in this slice. +The integrated lookup deadline is currently fixed at 1000 ms; the proposed +configuration key below is not wired into Sender yet. Complete conversion +coverage gates release, not the first integrated test. Until that +coverage is complete, this is an unreleased development increment: unsupported +schema-mode inputs fail explicitly rather than falling back to server conversion. +The [conversion backlog](#conversion-backlog-server-source-audit) inventories +every public input family against the current server, including missing identity +setters, unsupported pairs and unresolved compatibility boundaries. +The [implementation journal](schema-aware-sender-journal.md) +records decisions, iteration scope and verification evidence. + +Iteration 2.16 is accepted: two real-server tests create file-backed backlog +through public Sender setters, withhold data from the server and forcibly stop +the producer JVM. A fresh Sender replays byte-identical QWP payloads and stores +exact A/C rows after rejecting partial B. Coverage includes UUID/SYMBOL values, +two pinned UUID-to-VARCHAR schema generations, and reopening after ACK without +replaying acknowledged rows. The 230-test server gate and three seeded repeats +passed against the unchanged 2.15 client artifact. This increment changes only +tests and documentation; it proves neither power-loss durability nor a stronger +delivery guarantee, and does not close the release gaps above. D031 records +the evidence and review findings. + +Iteration 2.17's first repair group is accepted: legacy conversion comparisons +now send unflagged source-typed QWP frames instead of accidentally using schema +conversion on both sides. Original expected values are preserved, with explicit +wire assertions for source types, values and nulls. All 25 tests in the four +affected suites, the 251-test regression selection and three seeded repeats +passed. The original 333-test selection remains red with 56 failures/errors, +down from 58 known contract/fixture failures. Two additional disk-space failures +were resolved through the test JVM's startup configuration, not changed +expectations. No production changes or Sender opt-out were introduced. D032 +records the evidence and the next functional priority: missing-table/column +inference through the existing binding and buffer ownership. + +Iteration 2.18 is accepted for confirmed-missing inference in already-supported +setter families. Missing tables and columns use native inferred types in the +existing buffer; server-provided types still govern known columns. No separate +schema registry or new stored state was added. Same-row duplicates are ignored +before type checking; later-row inferred type conflicts use the existing +schema-refresh path. New metadata never relabels completed rows. Server creation +policy and authorization remain authoritative. +The client gate passed 414 tests plus two packaging checks; the server gate +passed 261 tests, with three seeded repeats of 16 integration/recovery tests. +The original 333-test selection has 54 failures/errors, down from 56, with no +new failing methods. D033 records the implementation, review corrections and +remaining diagnostic/conversion work. This does not complete unsupported input +families or make the client release-ready. + +Iteration 2.19 is accepted as a test-only diagnostic repair. Seven tests now +assert typed local errors and observable recovery. Independent legacy frames +still verify server rejection of BINARY-to-text/SYMBOL. The mismatch test waits +for ACK feedback before asserting the conversion error. No production behavior +or client artifact changed. The 268-test regression gate and three repeats of +23 integration/recovery cases passed. The unchanged 333-test selection has +47 failures/errors: exactly the seven repaired methods disappeared, with no +new failing methods. The remaining set is 38 missing-conversion cases, five +approved schema/legacy expectation differences and four obsolete fixtures. +D034 records evidence and a pending source-null decision for future text +conversions; this iteration does not approve another compatibility exception. + +Iteration 2.20 is accepted for source-null characterization, not conversion +implementation. Three real-server tests prove bitmap-dependent LONG/UUID text +results and current public Sender rejection/recovery across all five pairs. +The bitmap-present LONG result is non-null text `"null"`, correcting the earlier +decimal-minimum prediction. The 271-test regression gate and three repeats of +26 cases passed; the original 333-test selection retains the exact same 47 +failing methods. Production code and the client artifact are unchanged. +D035 records the evidence. The [text-null rules](#source-null-rules-for-text-output) +were proposals at that checkpoint; D039 subsequently approves LONG only. + +The [remaining-failures plan](schema-aware-sender-remaining-failures-plan.md) +assigns all 47 failing methods to small, independently tested increments: +nine expectation/fixture repairs, then 38 conversion cases. It preserves +independent legacy tests and requires public Sender E2E coverage in every +increment. D036 records the source audit and sequencing; this is a plan, +not another accepted implementation iteration or the full release checklist. + +Iteration 2.21 accepts phase 1a of that plan. Three public Sender Boolean tests +now assert target-native omissions; two floating-point legacy diagnostics send +independent unflagged frames. A new public Sender test preserves NaN and +positive-2^63 boundary/recovery coverage. All 25 focused tests, the expanded +277-test green gate and three repeats of 32 cases passed. The original 333-test +selection has 42 failing methods: exactly the five repaired cases disappeared, +with no new failure. No production code or client artifact changed. D037 records +the evidence. Four direct-buffer fixtures and all 38 missing-conversion cases +remained after that increment. + +Iteration 2.22 accepts phase 1b: the four direct-buffer fixtures now send +standalone unflagged legacy frames, preserving their original stored values. +They assert GeoHash precision/bitmap/packed bytes, literal named LONG designated +timestamps in both units, and legacy ACK framing. Public `at()` coverage remains +separate. All 37 focused tests, the 281-test green gate and three repeats of +36 cases passed. The original 333-test selection has 38 failing methods: +exactly the four repaired cases disappeared, with no additions. Phase 1's nine +test repairs are complete; no production code or client artifact changed. +Native public GeoHash conversion and all other missing conversions remain +backlog items. D038 records that checkpoint, before the LONG text-null decision. + +Iteration 2.23 accepts LONG-to-STRING/VARCHAR/SYMBOL in ordinary Sender. One +binding-local formatter uses existing target buffers; no server production or +send-time change is needed. `Long.MIN_VALUE` is an effective target-null write +in schema mode, with legacy behavior unchanged. Exact wire, shared vectors, +SQL, rollback, schema rebind and replay checks pass: 420 client tests plus two +packaging checks, 16 focused server tests, 288 server regressions and three +43-test seeded repeats. The original 333-test selection now has 35 errors: +exactly the three LONG-to-text cases disappeared, with no new failing method. +D039 records evidence and two stale rejection-oracle migrations. + +Iteration 2.24 accepts UUID-to-STRING/VARCHAR with the separately approved +both-limbs-MIN target-null rule. Every other pair, including either single MIN +limb, becomes canonical lowercase text. One lazy binding-local formatter feeds +the existing VARCHAR buffer; native UUID, legacy behavior and UUID-to-SYMBOL +rejection remain unchanged. Validation passes 426 client tests plus two packaging +checks, 21 focused server tests, 293 server regressions and three 48-test seeded +repeats. The original 333-test selection now has 33 errors: exactly the two +UUID-to-text cases disappeared, with no additions. Native UUID performance +checks found no material regression; they do not measure text throughput. D040 +records evidence and fixture corrections. Floating-point text-null rules were +still undecided at that checkpoint. + +Iteration 2.25 accepts FLOAT/DOUBLE-to-STRING/VARCHAR/SYMBOL with NaN as an +effective SQL NULL. Finite values, signed zero and infinities match the server; +FLOAT widens before formatting. A schema-only Ryu port preserves legacy Numbers. +Validation passes 432 client tests plus two packaging checks, 29 focused server +tests, 301 server regressions and three 56-test seeded repeats. The original +333-test selection has 30 errors: only the three DOUBLE-to-text cases disappeared. +Independent differential checks cover approximately two million non-NaN inputs. +Performance checks caught an inlining regression; separating text dispatch +removed the extra native DOUBLE instruction cost. D041 records the evidence, +remaining measurement limits and fixture corrections. Phase 2 is complete. + +Iteration 2.26 accepts LONG-to-ordinary-TIMESTAMP/TIMESTAMP_NS as raw counts in +the target unit. The existing LONG setter needs only two allowed targets and +two switch labels sharing its raw append. MIN remains an effective SQL NULL; +independent legacy frames confirm both bitmap contexts also store timestamp +NULL. Native LONG inference and named designated-timestamp rejection are +unchanged. Validation passes 437 client tests plus two packaging checks, +41 focused server tests, 305 regressions and three 68-test seeded repeats. +The original 333-test diagnostic has 29 errors: only LONG-to-TIMESTAMP was +removed from the prior 30. Native LONG measurements detect no regression; +they do not measure timestamp throughput. D042 records the evidence and a +corrected exact-wire fixture error. + +Iteration 2.27 accepts STRING-to-ordinary-TIMESTAMP/TIMESTAMP_NS. A fixed parser +preserves the server's eleven formats, UTF-8 replacement/ASCII-byte input view, +calendar arithmetic and timezone rules; nano targets check the multiplication +after microsecond parsing. Java-null input writes a bitmap null, while non-null +text that parses to MIN remains a present raw timestamp. No general compiler, +SQL-engine change, legacy change or new replay path is introduced. +Validation passes 442 client tests plus two packaging checks, 47 focused server +tests, 311 regressions and three 74-test seeded repeats. The original diagnostic +has 28 errors: only STRING-to-TIMESTAMP was removed from the prior 29. +The shared corpus has 110 literal cases; 214,030 public binding/encoder checks +per locale agree with the server in fresh en_US/tr_TR processes. Reference +century, locale and tzdb data still come from each process's runtime; this does +not guarantee agreement across different runtime data. Native STRING-to-VARCHAR +measurements show a consistent roughly 2–3% cycle/time cost, below the declared +5% guard, with effectively unchanged instructions. D043 records the findings, +corrections and measured trade-off. + +Iteration 2.28 accepts timestamp-to-STRING/VARCHAR. These writes reject +out-of-range values locally with INVALID_VALUE. NANOS are divided by 1000 toward +zero; MICROS through DAYS and Instant are converted to micros with range-correct +checked arithmetic before millisecond formatting. Instant keeps the existing +microsecond rounding. Preserve present MICRO MIN as empty text and NANO MIN as +finite timestamp text; this is not the LONG/NaN null policy. Old-server/legacy +behavior remains unchanged. One fixed formatter reuses the existing text sink; +no protocol, replay or SQL-engine change is needed. Validation passes 447 client +tests plus two packaging checks, 53 focused server tests, 317 regressions and +three 80-test seeded repeats. All 160,202 public binding/encoder comparisons +match the server formatter and checked-range oracle. The original 333-test +diagnostic now has 26 errors, removing only its two timestamp-to-text cases. +No repeatable native timestamp-write regression is detected by the performance +guard; this does not measure formatter throughput. D044 records the evidence. + +Iteration 2.29 accepts native CHAR and STRING-to-CHAR using existing two-byte +storage. Native values retain all 16 bits. Text takes the first BMP character, +ignoring the remainder; empty/NUL/supplementary-first text becomes code unit +zero, while malformed UTF-16 becomes '?', matching existing client replacement +and server decoding. Java null is bitmap-null; present zero remains wire-present +but reads as SQL NULL. CHAR-to-text remains separate missing work. Validation +passes 452 client tests plus two packaging checks, 59 focused server tests, +323 regressions and three 86-test seeded repeats. All 65,549 public differential +checks match the server. The unchanged original 333-test diagnostic has 24 +errors, removing only its two CHAR cases. The native string-write performance +guard passes. D045 records the corrected refresh catch and full evidence. + +Iteration 2.30 accepts STRING-to-LONG256 using four local longs and existing +32-byte storage. Match the server's lowercase `0x` prefix, 2–64 hex digits in +complete byte pairs and right-to-left limb order. Java null/omission is null; +the literal with four Long.MIN_VALUE limbs is INVALID_VALUE. Parse completely +before append. Native `long256Column` remains unsupported, and missing string +columns still infer VARCHAR. Validation passes 458 client tests plus two +packaging checks, 63 focused server tests, 327 regressions and three 90-test +seeded repeats. All 40,055 public binding/encoder differential checks match +the server oracle. The unchanged original 333-test diagnostic has 23 errors, +removing only STRING-to-LONG256. The native STRING-to-VARCHAR regression guard +passes; this does not measure LONG256 throughput. D046 records the evidence. + +Iteration 2.31 accepts STRING-to-GEOHASH using the existing base32 parser and +precision field. Parse at most 12 characters, require enough bits and truncate +to the target precision. Empty text and Java null become bitmap null. Always +emit the bitmap for schema-directed nullable GEOHASH columns, even with no null +rows: otherwise valid byte-aligned maximum values are mistaken for nulls. +Legacy and low-level non-nullable encoding remain unchanged. Only canonical +packed GEOHASH types are recognized; unknown flags retain typed rejection. +Native GEOHASH setters remain unsupported and missing strings still infer +VARCHAR. Validation passes 464 client tests plus two packaging checks, 67 +focused server tests, 331 regressions and three 94-test seeded repeats. All +7,210 public differential cases match. The original 333-test diagnostic has +22 errors, removing only STRING-to-GEOHASH. Scoped native STRING-to-VARCHAR +checks measure 1.46–1.80% more append time and 0.17–1.15% more encoding time, +below the declared regression threshold; this is not GEOHASH throughput +evidence. D047 records the null-framing correction, flagged-type regression +and independent reviews. + +Iteration 2.32 accepts all three native decimal setters into every +DECIMAL8/16/32/64/128/256 target. This is 18 logical pairs represented by nine +physical source/target wire-width pairs: small decimal targets use DECIMAL64 +wire storage while retaining their own precision and scale. Conversion copies +the caller's value, rescales exactly, checks target precision and physical +storage width, and appends target-width limbs with the target scale retained +across reset. Public null setters remain no-ops; confirmed MISSING and legacy +paths keep source-width inference and legacy scale behavior. Tests cover caller +immutability, bitmap-only decimal nullness, exact wire/SQL results, failed-row +rollback and schema-generation transitions. Validation passes 472 client tests +plus two packaging checks, seven focused and 338 canonical server tests, and +three 101-test seeded repeats. The unchanged 333-test diagnostic has 18 errors: +only its four intended phase-5a methods disappeared. D048 records the 60-row +shared corpus and the bounded performance evidence. The 12-run legacy A/B guard +passes with +0.001% to +0.002% instructions and -0.47% to +1.99% cycles. Six +candidate-only nine-physical-pair runs measure median absolute cost of 32.91 ns, +182.03 cycles and 1140.18 instructions per value; they make no network, ingestion +or general-throughput claim. Phases 5b–5e remain; this does not establish complete +conversion coverage or release readiness. One low-priority diagnostic issue is +deferred: scale-up overflow has the correct `INVALID_VALUE` reason and rollback, +but its detail says the value cannot be rescaled exactly instead of saying that +it overflowed. Do not add another conversion branch solely to refine that text. + +Iteration 2.33 accepts LONG into every DECIMAL8/16/32/64/128/256 target. The +client treats the integer as a scale-zero value, rescales it exactly to the +target scale, checks declared precision and physical storage width, and writes +the direct target decimal wire type. `Long.MIN_VALUE` is a stable schema-mode +source null: it emits a decimal bitmap null and locks the target scale, regardless +of whether another row omits the column. Confirmed MISSING and legacy paths are +unchanged; the latter retains its existing bitmap-dependent sentinel behavior. +The implementation reuses the decimal column's existing scratch value and the +phase-5a validation/append tail; it adds no schema state, conversion registry or +send-time transformation. Validation passes 475 client tests plus two packaging +checks, seven focused and 345 canonical server tests, and three 108-test seeded +repeats. The unchanged 333-test diagnostic has 14 errors: exactly the four +LONG-to-decimal methods disappeared, with no additions. D049 records the 39-row +shared corpus, schema-generation test, compatibility characterization and +bounded performance evidence. The 12-run LONG-to-LONG A/B guard detects no +repeatable 5% cycles/instructions regression; this is not decimal, network or +ingestion throughput evidence. Phases 5c–5e remain, so release coverage is not +complete. + +Iteration 2.34 accepts STRING into every DECIMAL8/16/32/64/128/256 target. The +client parses at the target precision and scale with the server's `strict=false`, +`lossy=false` rules, including exponent underscores, then writes the QWP decimal +wire type for the target storage class through the existing decimal append tail; +DECIMAL8/16/32/64 targets share QWP's DECIMAL64 wire type. The shared +parser now uses the current server's leading-zero and full-scale precision +calculation. Exact, optionally signed `NaN` and `Infinity`, after supported +suffix processing, and Java null become target-scale bitmap SQL nulls. +Malformed special-value prefixes and extreme nonzero integer-boundary exponents +fail locally. +These rules deliberately do not copy three server defects: accepting +special-value prefix garbage as null, narrowing a special null to numeric zero +for small decimals, and throwing an array-bounds exception for extreme exponents. + +Confirmed MISSING columns and legacy mode remain VARCHAR and retain server-side +conversion. The implementation reuses the per-column Decimal256 scratch and the +phase-5a append tail; it adds no state, framework, protocol, server production, +SQL or send-time transformation. The 55-row shared corpus contains 19 values, +24 invalid inputs and 12 nulls; both copies have SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`. +Client red validation first failed eight tests; final focused validation passes +eight tests plus two packaging checks, all-decimal validation passes 463 tests, +and the canonical gate passes 941. The installed candidate JAR has SHA-256 +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +A 3,601,944-case differential passes ordinary server parity and records 72 +intentional malformed-prefix corrections. Updating the shared parser's precision +metadata required correcting 19 stale assertions in 16 existing tests; their +values and scales did not change. + +The parser correction is global, not limited to schema mode: direct +`Decimal64`, `Decimal128` and `Decimal256` string parsing now uses the corrected +precision metadata, rejects malformed special-value prefixes and reports +extreme nonzero exponents as `NumericException`. This deliberate compatibility +change removes unsafe or undocumented behavior for legacy and standalone parser +callers as well as enabling schema-directed conversion. + +Server validation passes seven focused and 352 canonical tests. The unchanged +original 56-test conversion class now has ten errors: exactly the four +STRING-to-decimal +methods disappeared, with no additions. Three seeded repeats pass 115 tests and +22 archived XML reports each. A 12-run balanced fixed-work native +STRING-to-VARCHAR guard uses checksum `15640344951909530213`; paired basic +instructions change by -0.001% to -0.000% and cycles by -1.028% to -0.007%, with +no repeatable 5% regression. This is not decimal, network or ingestion throughput +evidence. D050 records the bounded acceptance. Phases 5d–5e remain, so release +coverage is not complete. + +Iteration 2.35 accepts FLOAT and DOUBLE into every +DECIMAL8/16/32/64/128/256 target. FLOAT is widened exactly to DOUBLE, then the +existing schema formatter produces the shortest round-tripping decimal and the +existing decimal parser enforces the target precision and scale without rounding +or truncation. Finite values whose natural scale or precision does not fit fail +locally with `INVALID_VALUE`; NaN and both infinities become target-scale bitmap +SQL nulls. Confirmed MISSING and legacy mode retain native FLOAT/DOUBLE bytes and +server-side conversion. + +The implementation adds only a decimal branch to the existing cold nonnumeric +floating helper and reuses the per-column Decimal256 scratch plus the phase-5a +target-width append tail. It adds no state, parser, framework, protocol, server +production, SQL or send-time transformation. Moving the branch out of the common +numeric helper was necessary: the first implementation pushed that method past +HotSpot's hot inlining threshold and increased native FLOAT/DOUBLE instructions +by about 3.8–4.7%. After the split, six counterbalanced pairs per native path show +median instruction changes of +0.0001% for FLOAT and +0.0037% for DOUBLE; median +cycle changes are -0.83% and -1.04%. This is a native binding regression guard, +not floating-to-decimal or ingestion throughput evidence. + +The shared client/server corpus has 60 rows: 37 values, 17 invalid values and six +nulls, with SHA-256 +`366a889719c6e7b688ff00e1bdfb95e273ab192508a1094dac5312c3d899611d`. +A 5,996,868-case differential matches the server's accepted values, target scale +and all four limbs. Client focused validation passes 63 tests plus two packaging +checks and the canonical gate passes 944; an expanded 1,269-test regression gate +also passes. The candidate and installed JAR SHA-256 is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +Server validation passes the eight-test focused gate and 360-test canonical +gate. At that checkpoint, the unchanged original 56-test conversion diagnostic +had six errors, all assigned to phase 5e, with no additions. Three seeded repeats +passed 123 tests and 23 selected XML reports each. D051 records the test-fixture +cleanup and bounded phase-5d acceptance; the phase-5e acceptance below +supersedes its remaining-failure count. + +Iteration 2.36 accepts native Decimal64, Decimal128 and Decimal256 values into +STRING and VARCHAR. The binding promotes each non-null value into one lazily +reused Decimal256 scratch, applies the unsigned QWP scale byte, formats into the +existing reused text sink and appends `TYPE_VARCHAR`. This mirrors the server's +per-value scalar conversion, including full physical Decimal64/128 coefficients, +without a new validation policy, conversion registry, protocol state, server +change, SQL change or send-time transformation. Public null sentinels remain +setter no-ops. + +Schema mode intentionally formats each value independently, so mixed native +widths and scales can share a known text target. Legacy inference still fixes +one native width and scale for a whole column block and may reject or rescale a +later value. Preserving that irrelevant intermediate restriction after the text +target is known would add state and make conversion depend on neighboring rows. + +The 24-row shared corpus covers all six source/target pairs and both public null +forms. A second public boundary test covers the full physical Decimal128 value +`2^127 - 1` and unsigned scale-byte wrapping. The production/server differential +passes 3,000,294 full-physical and arbitrary-scale comparisons. Exact-current +client gates pass the 46-test public integration class, including four phase-5e +tests, plus 950 canonical tests, 1,275 broad tests and two packaged-JAR tests. +The candidate and installed JAR SHA-256 is +`e06759605ee3d1cf01c3277ba13531b0172e42138042c1f2f49c2db49e04b22d`; +only `QwpSchemaBinding.class` differs from the Phase-5d JAR. + +The unchanged server conversion diagnostic now passes all 56 methods, resolving +the original 47 failures. The accumulated server gate passes 363 tests; three +seeded repeats pass 126 tests each. The identity-path performance guard finds a +0.71–1.38% instruction increase and cycle medians from -1.93% to +1.13%, below +the 5% rejection threshold. It makes no speed, network or ingestion-throughput +claim. D053 and the iteration-2.36 run manifests record exact evidence. + +## Automatic negotiation and compatibility + +The new client works with old servers and automatically enables schema-directed +encoding when the server supports it. There is no configuration switch or +alternate write API to disable it on a supporting server. The earlier +`schema_encoding` and `schema_validation` options are removed from the proposal. + +Keep the base protocol at QWP v1: the existing magic, version byte and version +negotiation do not change. Negotiate the schema extension during WebSocket +upgrade using these proposed headers: + +``` +Client request: X-QWP-Request-Schema: true +Server response: X-QWP-Schema: enabled +``` + +The new client always requests the extension. A successful upgrade without the +response header selects legacy behavior, unless this sender has previously +confirmed schema support. Explicit confirmation enables schema requests, +schema-directed encoding and extended responses. An unrecognized response +header is a negotiation error, not evidence of an old server; neither are +connection failures, timeouts or authorization failures. + +The upgrade is one-way: once a sender confirms schema support, it assumes all +subsequent connections support the extension too. Require confirmation on +reconnect and background replay connections; never switch back to legacy mode. +Moving an upgraded sender to an old server is unsupported. Retain pending data +and report the capability mismatch under the existing reconnect policy. There +is no send-time transformation, metadata stripping or downgrade encoder. + +| Client | Server | Behavior | +|---|---|---| +| Existing client, no schema request | New server | Existing QWP v1 framing and behavior | +| New client, not yet upgraded | Old server | Existing inferred-type QWP v1 behavior | +| New client, always requests schema | Supporting server | QWP v1 with the schema extension; one-way upgrade | +| New client, already upgraded | Old server | Unsupported endpoint; retain pending data, no downgrade | + +The server enables the extension only when requested and confirmed. Connections +without negotiation receive no schema frames or extended ACKs/NACKs. Existing +clients need no changes. Clients may be upgraded before servers. Once schema +support has been confirmed, failover destinations must also support it; mixed +old/new endpoints are not supported for that sender after the upgrade. + +An existing legacy row finishes in legacy mode. The producer adopts schema +mode at the next actual row boundary, seals completed legacy rows and replaces +the inferred layout before binding new rows. Pending legacy blocks retain +their original types and flag-clear framing, even if encoded or sent later. + +The extension defines one stable conversion contract, backed by shared test +vectors, rather than a separate conversion-version handshake or a dependency on +a particular server release. A schema alone does not specify parsing or +rounding rules. In schema mode, unknown target types fail explicitly when used, +without blocking writes to unrelated supported columns or falling back to +raw-value encoding. Schema lookup failures never disable schema mode. + +## Availability + +In schema mode, required lookups have a bounded deadline +(`schema_describe_timeout_millis`, proposed default 1000). Timeout, unavailable +metadata or request saturation throws a lookup error before accepting the +value. It cancels any partial row but does not permanently halt the sender. + +Known schemas can be used offline; unfamiliar tables require a live lookup. +Reconnect invalidates the lookup cache for future row starts, but does not reset +an upgraded sender to legacy mode, invalidate an already pinned row or rewrite +buffered and persisted blocks. Legacy mode retains existing offline buffering. + +Asynchronous construction may remain, but the first write waits for the initial +successful negotiation to select the mode within a bounded deadline. An offline +or unreachable server is not assumed to be old. In schema mode, required lookup +must also complete before accepting the value; in legacy mode, no lookup is +needed. This changes cold-start offline behavior, not the minimum server version. + +## Frame layout and replay compatibility + +Negotiation authorizes the extension; a persisted frame flag identifies the +actual data layout. Use `FLAG_SCHEMA = 0x02` without changing the QWP version byte: + +- Flag set: each table header includes its pinned schema identity, or an + explicit unknown identity. The server uses the extended table-header parser. +- Flag clear: table headers have the existing layout and no schema identities. + These are legacy-mode writes or legacy SF frames, accepted even on a + schema-negotiated connection. +- Flag set without successful negotiation: an extension-aware server rejects + the frame before processing table data. The client must never send it to an + endpoint that did not confirm support. + +In a flagged frame, each table header starts with the existing name-length +varint and UTF-8 name, followed by `identityKind:u8`. Kind 0 means unknown and +has no body. Kind 1 carries `tableId:i32 LE` and `metadataVersion:i64 LE`, both +non-negative. The existing row-count and column-count varints and column +definitions follow. Reject reserved kinds, truncated identities and negative +known fields. Flag-clear headers retain their exact old layout. + +`FLAG_SCHEMA` cannot be combined with `FLAG_CONTROL`, and schema-flagged frames +must contain at least one table. Table-less commit/dictionary traffic stays +flag-clear. The parser still uses inline column definitions to interpret the +values; schema identities never replace them or enforce a version precondition. + +Every frame containing schema-mode table blocks sets the flag, including writes +that use auto-creation. Legacy blocks keep flag-clear framing. Each frame uses +one table-header layout; keep legacy and schema-mode blocks in separate frames +during the one-way transition, preserving order and transaction boundaries. +Table-less dictionary/control traffic retains its own framing rules. + +Persist the flag with the schema identities and data bytes. Replay preserves +all of them, allowing legacy and extended data frames to coexist in sequence on +one negotiated QWP v1 connection. Do not infer the table-header layout solely +from connection state or relabel old frames with new flags or identities. + +Repeat negotiation on every reconnect, including background replay connections. +Before upgrade, legacy SF can drain to either an old or a supporting server. +After upgrade, require support even when the remaining frames happen to be +legacy. Recovery of pending extended SF data also requires support: establish +that requirement before replaying any part of that backlog, including a legacy +prefix. A fresh sender with only legacy SF negotiates normally. An incompatible +endpoint cannot drain a schema-required backlog: retain pending bytes and find +or wait for a compatible endpoint under the reconnect policy. Do not advance +watermarks or alter stored frames. + +ACK/NACK extension framing is selected by the negotiated connection, independent +of the data frame's flag. A negotiated connection may receive schema feedback +for legacy blocks too; their absent identities are treated as unknown. Ordinary +and durable-ACK delivery semantics remain unchanged. + +Keeping QWP at v1 does not make extended SF data readable by an older client. +Before downgrading the client binary, stop new writes and drain all pending +extended data, including background/orphan slots, with a supporting client and +server. Completion means acknowledgment under the configured ordinary or +durable-ACK policy, not merely a successful send or an attempted close. +Downgrading with pending extended data is unsupported. If draining fails, +retain the data and continue recovery with a supporting client; do not rewrite +frames, strip metadata, delete pending logs or advance watermarks to permit the +downgrade. This is an operational prerequisite, not a claim that older binaries +can detect and enforce it. + +## Schema lookup + +Each schema carries `(tableId, metadataVersion)`. The server already versions +table metadata; `tableId` distinguishes drop/recreate. The pair is an identity, +not a global ordering between table incarnations. + +The logical control exchange is: + +``` +DESCRIBE: requestId, tableName +SCHEMA: requestId, tableName, result, schema? +schema: tableId, metadataVersion, designatedIndex, columns[] +column: name, serverType, typeParams +``` + +The schema includes full target types and parameters: decimal precision/scale, +timestamp units, array element type/rank and geohash bits. +`designatedIndex` identifies the target of `at(ts)`; -1 means none. +Names follow the server's comparison rules. There is no per-column +`acceptMask`: the extension's conversion contract and target type determine +the client converter. + +Results distinguish a known schema, a confirmed missing table and unavailable +metadata. Missing tables may follow auto-creation; unavailable metadata must +never be represented as a missing table. Permission denial has a distinct +machine-readable reason and must not trigger auto-creation. + +An account authorized to write a table must be able to obtain the ingestion +schema needed for that write without gaining SELECT access to its data. Apply +the table-write authorization to both DESCRIBE and schema feedback in ACKs/NACKs; +metadata being available is not sufficient permission to return it. Return only +the ingestion schema, not table data or unrelated privileged metadata. Retain +the existing authorization rules for creating missing tables and adding columns. +Verify write-only accounts and denial/revocation behavior as part of the server +lookup implementation. Cached metadata never substitutes for server-side write +authorization. + +The producer waits while the I/O thread services the connection and lookup. +Describes do not flush rows, consume data-frame sequence numbers or affect +ACK watermarks. + +Direct SCHEMA replies retain the full QWP control-frame header. Route that +envelope separately from ordinary and durable ACKs, then validate it strictly; +do not guess the response type by trying multiple decoders. The request ID +identifies the requested table; the current direct reply does not repeat its +name. Named ACK/NACK feedback reuses only the inner schema payload. + +## Column binding and conversion + +This section describes schema mode. Legacy mode keeps existing binding, +conversion and missing-value behavior unchanged. + +The sender maintains a sparse layout containing only columns being written. +For each known column, its binding contains the target type and parameters, +the chosen QWP wire representation, and converters for the supplied input +types. Schema resolution and converter selection can be cached; conversion +and value-dependent checks happen on every effective write. + +Server types and wire types are not necessarily one-to-one. For example, a +small server DECIMAL may use a DECIMAL64 wire representation. The client must +still enforce the server column's precision and scale before buffering. + +For text and symbol inputs, STRING and VARCHAR targets both use the existing +VARCHAR wire representation; a SYMBOL target uses symbol encoding. Reuse the +existing UTF-8 serialization rules: preserve case, whitespace, embedded NUL and +valid Unicode, and replace each unpaired UTF-16 surrogate with literal `?`. +Null and empty text remain distinct. Symbol dictionary IDs are representation, +not value identity: distinct source strings may encode to the same text after +surrogate replacement. No new dictionary owner or send-time type conversion is +needed for these text-to-text pairs. + +For a BINARY target, binary setters copy opaque bytes without interpretation. +`stringColumn` writes the same UTF-8 bytes it would send through the legacy +VARCHAR representation, including NUL and the surrogate-replacement rule above; +this is not hex or base64 parsing. All three binary input forms (byte array, +DirectByteSlice and native pointer/length) select TYPE_BINARY. Effective null +byte-array/slice arguments report `INVALID_VALUE`; string null and omission +produce target NULL. Empty input remains a present zero-length BINARY value, +distinct from NULL on the current server. + +Native binary lengths must be 0..Integer.MAX_VALUE, with a nonzero pointer for +nonempty input. Reject invalid lengths before reading memory. The caller must +provide readable memory for the duration of the setter; the binding retains +neither the source address nor the slice. A zero pointer with zero length is +valid empty input. First-value-wins precedes these checks. Binary input into +other target types is not implemented, including the unresolved parser paths +listed below. + +String and binary byte appends preserve their previous data and offset positions +if a RuntimeException or Error occurs before publication. The same exception is +rethrown; the caller still cancels the partial row and rolls back its new column +definitions. This repairs exceptional cleanup in the shared buffer, not just +schema mode, without changing successful wire bytes. It does not claim recovery +from invalid native addresses or fatal process faults. + +Boolean inputs use the target representation: BOOLEAN values, numeric `0`/`1` +for BYTE/SHORT/INT/LONG/FLOAT/DOUBLE, and lowercase `false`/`true` text for +STRING/VARCHAR. BOOLEAN-to-SYMBOL is not a supported server conversion. +Text-to-BOOLEAN accepts only `0`, `1` and ASCII-case-insensitive `true`/`false`. +Do not trim whitespace, apply Unicode case folding or silently treat an invalid +token as false. Invalid text reports `INVALID_VALUE` before append. +Explicit null and omission follow the target rule below: BOOLEAN stores false, +BYTE/SHORT store zero, and nullable numeric/text targets store NULL. This does +not make unsupported input/target pairs valid. + +Unlike the current inferred layout, a known column is not bound to the first +setter's input type. Different setters may feed the same target buffer across +rows, provided their conversions are supported. Existing name validation and +setter no-op rules remain. In schema mode, first-value-wins is applied before +conversion even when the duplicate uses a different setter; ignored values +are not converted. Legacy mode ignores same-type duplicates but throws on +cross-setter type mismatches. + +Omitted ordinary columns and explicit nulls use the target column's missing-value +representation: SQL NULL where supported, otherwise the target's native +missing-value/default representation. Do not synthesize a source-type default +and then convert it. A setter that treats null as a no-op still does not bind or +write a value; any resulting omission follows the target rule. Inferred columns +use their inferred target type until a server-backed binding replaces it. + +This intentionally changes some existing results. After a boolean setter binds +the old client layout, an omitted value sent to a STRING column currently becomes +the text "false". With a STRING target binding, omission instead becomes NULL; +an explicitly supplied false still converts to "false". Missing-value behavior +must not depend on which input setter was used in an earlier row. The designated +timestamp remains the documented exception: `atNow()` uses server-assigned time. +Document this behavior change and test both negotiated modes and old clients. + +For LONG input to a numeric target in schema mode, `Long.MIN_VALUE` means null +before conversion, regardless of other rows in the block. Append the target's +missing-value representation: zero for BYTE/SHORT, NULL for INT/LONG/FLOAT/DOUBLE. +Apply duplicate suppression and validate the supported input/target pair first; +this rule does not make an unsupported conversion such as LONG-to-UUID valid. + +For LONG input to STRING, VARCHAR or SYMBOL, apply the same source sentinel +decision: `Long.MIN_VALUE` writes target SQL NULL before formatting, independently +of other rows. Other values become canonical signed decimal text. This approved +schema-mode rule deliberately differs from legacy bitmap-present MIN, which +can become the literal text "null". An explicit MIN is an effective first write, +not a no-op; an ignored duplicate cannot replace it. + +This deliberately differs from legacy QWP's block-dependent sentinel handling. +With a null bitmap present, a supplied LONG minimum can be rejected by a narrow +integer target or stored as a finite FLOAT/DOUBLE; without the bitmap, it is +treated as null. Schema mode does not reproduce that inconsistency. Legacy +writes and existing legacy SF retain their behavior. No end-of-batch or +send-time conversion is introduced to imitate it. + +Conversion parity with QWP ingestion for supplied values is a release +requirement, except for the deliberate missing-value and LONG source-null rules +above and the LONG text range, floating-point and timestamp rules below. +Coverage includes: + +- String parsing into UUID, numbers, timestamps, decimals and other supported + targets; conversions to text must preserve the server's formatting rules. +- Numeric range, fractional-value, precision-loss, rounding, null-sentinel + and special floating-point behavior. +- Timestamp unit conversion and overflow; decimal precision and rescaling. +- Each non-null array's supported element type and rank, including wrapper and + Java overloads. Rank is not fixed by today's client layout. Do not promise + element conversion: the server copies array elements, and its exposed + `LONG_ARRAY` path has unresolved validation gaps described in the backlog. +- Geohash precision: native GEOHASH input requires an exact bit-count match, whereas + string parsing follows the server's geohash conversion rules. + +Do not substitute a convenient library parser or Java cast without proving +equivalent behavior. Use shared conformance vectors across server and clients, +checking acceptance, rejection and resulting stored values. An implementation +with missing converters must report that limitation, not claim a schema +mismatch or silently pass the value through to the server. + +For STRING input to LONG in schema mode, reject text outside the mathematical +signed-long range before append. This is an approved compatibility exception: +legacy ingestion accepts some overflowing decimal strings and stores wrapped +values, for example `"21000000000000000000"` becomes `2553255926290448384`. +Legacy parsing and ingestion retain that behavior. Preserve the server's other +LONG text grammar; do not use this range fix to broaden or tighten syntax. +An accepted string representing LONG minimum remains a supplied target +sentinel that reads SQL NULL, not a LONG setter's source-null conversion. + +For FLOAT/DOUBLE input to supported numeric targets, NaN means source null, +independent of other rows. Apply duplicate suppression and validate the supported +input/target pair first, then append the target's missing-value representation. +NaN payload and sign bits are not retained as values. Integer targets require a +whole value within the target's mathematical range; fractions, infinities and +out-of-range values report `INVALID_VALUE` before append. The LONG upper bound +is exclusive positive `2^63`, not a floating-point approximation of +`Long.MAX_VALUE`. Exact negative `2^63` and INT minimum remain representable +values that read as the target's SQL-null sentinel, matching server behavior. + +These are approved schema-mode exceptions: legacy NaN-to-integer writes use +the target's missing value without a null bitmap but fail when another row's +omission causes a bitmap; legacy positive `2^63` to LONG silently stores +`Long.MAX_VALUE`. Legacy ingestion and replay keep those behaviors. FLOAT/DOUBLE +targets still follow server widening/narrowing, including signed zero, +infinities, subnormal rounding and finite DOUBLE overflow to FLOAT infinity. +Do not add batch-dependent or send-time conversion. + +For typed microsecond/nanosecond inputs into ordinary timestamp columns, +nanoseconds-to-microseconds division truncates toward zero, including negative +values. Microseconds-to-nanoseconds multiplication rejects values outside the +server's representable range before append. Matching units preserve the bits. +The primitive timestamp input does not inherit the LONG-input null rule: +`Long.MIN_VALUE` overflows when scaling micros to nanos, becomes a finite value +when dividing nanos to micros, and reads as SQL NULL when stored unchanged. +Explicit nulls and omissions still use the target's null representation. +Test these cases both with and without another omitted row in the block. + +For primitive timestamp inputs, support NANOS, MICROS, MILLIS, SECONDS, MINUTES, +HOURS and DAYS. Convert directly to the target unit with checked arithmetic; +values outside its range report `INVALID_VALUE` before append. Other units +report `UNSUPPORTED_FEATURE`, and a null unit reports `INVALID_VALUE`. + +For `Instant`, preserve all nanoseconds in a TIMESTAMP_NS target. For TIMESTAMP, +retain the existing formula `epochSeconds * 1000000 + nanoFraction / 1000`; +this differs from primitive nanos-to-micros truncation for negative sub-micro +instants. Check the final combined value's range: an intermediate multiplication +must not reject an otherwise representable negative instant. A null `Instant` +reports `INVALID_VALUE`, not an implicit timestamp null. Duplicate suppression +precedes argument validation and conversion for both overloads. + +These are intentional schema-mode changes: legacy QWP scales larger units with +unchecked arithmetic and converts `Instant` to micros even for a nano target. +Legacy mode keeps that behavior. Shared vectors must test the new contract and +explicit legacy comparisons must document the changed overflow and precision. + +`at(ts)` converts to the pinned designated timestamp's unit before append and +row commit, including when the timestamp column reference is cached. A known +schema without a designated timestamp rejects explicit `at(ts)`. +`atNow()` resolves a schema if no earlier effective setter did so, then keeps +server-assigned timestamp behavior. + +Confirmed missing tables or columns retain client-side inference and existing +auto-create policy. Inferred bindings are distinct from server-backed bindings; +the client cannot promise conversion validity against a target that does not +yet exist. Successful writes return the resulting schema. + +Inference uses the existing buffer's column definitions, not fabricated server +metadata. The first effective supported setter establishes the native type: +BOOLEAN, LONG, FLOAT, DOUBLE, VARCHAR for text, SYMBOL, UUID, BINARY, CHAR, or a +timestamp. Primitive NANOS infers TIMESTAMP_NS; other supported units and +`Instant` infer TIMESTAMP. Only a missing table may infer the unnamed designated +timestamp. A conflicting type in a later row is a schema rejection and gets +the same one-refresh opportunity as a known-column rejection; a duplicate +within the current row is ignored. Failed-only definitions are removed by the +existing row rollback. Unsupported setter families remain unsupported even +when the table or column is absent. + +## Conversion backlog (server-source audit) + +Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage +refreshed through iteration 2.36 against the working trees, including uncommitted changes: client base +`981bdb02a471f3b290c89b8e78cbc422610e329e`, server base +`12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; +runtime evidence for implemented slices is recorded separately in the journal. +Read the actual QWP cursor selection, target dispatch and +value routines together; neither a shared switch branch nor an SQL cast proves +that QWP accepts a pair. + +### Complete input-to-target inventory + +The table covers effective, non-null values into existing **ordinary** columns. +"Implemented" records `QwpSchemaBinding` coverage; Sender integration is +described separately below. +An accepted null, ignored duplicate or no-op does not establish support for a +non-null conversion. Known compatibility exceptions above still apply. + +- **Numeric** means BYTE, SHORT, INT, LONG, FLOAT and DOUBLE, separately. +- **Text** means STRING and VARCHAR, separately; SYMBOL is always listed apart. +- **Timestamps** means TIMESTAMP (microseconds) and TIMESTAMP_NS (nanoseconds), + separately (`TIMESTAMP_NANO` in server code). +- **Decimals** means every DECIMAL8/16/32/64/128/256 target, with its declared + precision and scale. **Geohash** means every valid 1..60-bit target. +- Input names refer to public setters. `stringColumn` emits `TYPE_VARCHAR`; + there is no separate `TYPE_STRING` wire code. `symbol` emits dictionary-backed + `TYPE_SYMBOL`, not text-parser input. BYTE/SHORT/INT setters emit their own + wire types; they are not aliases for `longColumn`. + +| Public input (source wire type) | Implemented targets | Missing server-accepted targets | +| --- | --- | --- | +| `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | +| `byteColumn`, `shortColumn` (BYTE, SHORT) | None | Numeric, DATE, timestamps, text, SYMBOL, decimals; includes each input's identity pair | +| `intColumn` (INT) | None | Numeric, DATE, timestamps, text, SYMBOL, decimals, IPv4; includes INT identity | +| `longColumn` (LONG) | Numeric, text, SYMBOL, timestamps, decimals | DATE | +| `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | +| `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | +| `symbol` (SYMBOL) | Text, SYMBOL | None | +| `timestampColumn(long, unit)` (TIMESTAMP or TIMESTAMP_NANOS) | Timestamps, text | None | +| `timestampColumn(Instant)` (currently TIMESTAMP micros in legacy Sender) | Timestamps, preserving nanos for the schema-mode nano target; text | None | +| `uuidColumn` (UUID) | UUID, text | None | +| `charColumn` (CHAR) | CHAR | Text; non-ASCII text output needs the decision below | +| `ipv4Column(int)` and `ipv4Column(CharSequence)` (IPv4; text parsed locally) | None | IPv4, text | +| `long256Column` (LONG256) | None | LONG256, text | +| `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | +| `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | +| `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | None | All decimals, text; this is not the `stringColumn` parser path | +| `geoHashColumn(long, bits)` and `geoHashColumn(CharSequence)` (GEOHASH with source bits) | None | Geohash with exactly matching bit precision, text | +| `doubleArray` (all Java ranks and `DoubleArray`; DOUBLE_ARRAY) | None | DOUBLE array of the target rank; identity/shape validation, not element casting | +| `longArray` (all Java ranks and `LongArray`; LONG_ARRAY) | None | No supported conversion contract established; see the existing-server gap below | + +DATE also exists as a wire input, but there is no public WebSocket `dateColumn` +setter. The server accepts DATE-to-DATE and DATE-to-text only. Neither has a +binding entrypoint today; record this wire-only coverage without adding a new +public API just to mirror the protocol. DATE **targets** from existing integer +and string setters are real missing work in the table. + +The binding also lacks target representation selection for DATE, +IPv4 and arrays. It rejects extension parameters. +Adding a parser alone therefore does not complete these pairs: select the +target wire representation and pin its parameters before append. Preserve the +existing non-owning binding and buffer ownership. + +### Rules the missing conversions must cover + +These are server-source requirements to test, not permission to silently add +new casts or copy the unresolved behaviours below. + +- **Integer sources:** retain distinct BYTE/SHORT/INT/LONG input contracts. + Numeric narrowing checks range. DATE takes raw epoch milliseconds; ordinary + timestamps take raw counts in the target's unit, with no implicit unit + conversion. INT-to-IPv4 has its own null translation; it is not LONG-to-IPv4. + Integer-to-text/SYMBOL uses `Numbers.append(long)`. +- **Floating output:** text/SYMBOL formatting uses + `Numbers.append(cursor.getDouble())`, including widening FLOAT first. Do not + substitute `Float.toString`. Test signed zero, infinities, NaN and rounding. + The current server uses Ryu while the client still has the older formatter; + matching method names and default scale do not establish identical text. + Iteration 2.25 implements an isolated server-equivalent Ryu formatter; + the shared legacy formatter is unchanged. +- **Text parsing:** DATE uses `DateFormatUtils.parseDate`; timestamps use + `MicrosFormatUtils.parseTimestamp`, then checked multiplication by 1000 for + a nano target. Do not assume a separate nanosecond text parser. LONG256 uses + `Numbers.parseLong256`. VARCHAR-to-BINARY preserves the existing UTF-8 byte + payload, not hex/base64 decoding. CHAR takes the first decodable BMP character + and ignores the remainder; empty, malformed raw UTF-8 or supplementary-first + input produces NUL. Public malformed UTF-16 instead becomes '?', matching + existing client replacement before server decoding. Java null uses bitmap + null; present NUL is distinct on the wire but also reads as SQL NULL (D045). +- **Timestamp output:** both text targets use + `MicrosFormatUtils.appendDateTime`; nano wire values divide by 1000, truncating + toward zero, before formatting. Cover each primitive unit and `Instant` + separately. D044 separately approves checked normalization into legacy + source precision for text targets, without changing legacy mode. +- **Other text output:** UUID uses `Numbers.appendUuid`, LONG256 uses + `Numbers.appendLong256`, IPv4 uses `Numbers.intToIPv4Sink`, DATE uses + `DateFormatUtils.appendDateTime`, and decimal output preserves source scale. + CHAR uses the server's `putAscii` path; see the non-ASCII limitation below. +- **Decimals:** all three decimal wire widths reach every decimal target. + Decimal input rescaling must be exact: discarded nonzero digits cause + precision loss; scale-up/precision/storage overflow fails. Integer input is + scaled to the target; text is parsed at target precision and scale. + FLOAT/DOUBLE uses the server's `Numbers.doubleToDecimal(..., false)` path; + non-finite input becomes decimal null, and finite conversion overflow fails. + Do not replace its rounding with an assumed BigDecimal policy. Small server + decimals still need a supported wire width, such as DECIMAL64, while enforcing + the smaller target's precision/scale. All typed and text overloads need tests. + Iteration 2.32 implements and validates the three typed overloads into all six + decimal targets. Iteration 2.33 adds LONG into all six targets, treating + `Long.MIN_VALUE` as a target-scale bitmap null in schema mode. Iteration 2.34 + adds STRING into all six targets at the target precision and scale, preserving + exponent underscores and applying the stable special-value null rules above. + Iteration 2.35 adds FLOAT/DOUBLE into all six targets through the server's + lossless shortest-decimal path, with non-finite values encoded as decimal + nulls. Phase 5e converts non-null `Decimal64`, `Decimal128` and `Decimal256` + values targeting STRING or VARCHAR to plain fixed-point text before append. + Like the server, it first promotes the physical value to a reusable + `Decimal256`, applies the unsigned one-byte QWP scale, and then formats it. + This preserves valid source scales and trailing zeros, handles the full + physical Decimal64/128 range without the narrower formatters' precision + limit, emits `TYPE_VARCHAR` for both targets, and keeps public decimal null + sentinels as setter no-ops. + MISSING and legacy mode retain their native decimal wire type and source scale. + Other integer widths remain. + + Parity here is per setter value after schema binding, not a replay of the + legacy native-decimal column buffer. For a known text target, each row is + formatted independently and mixed Decimal64/128/256 inputs and scales are + accepted. Legacy inference instead fixes one native decimal width and scale + for the whole column block, rejecting a later width and rescaling or rejecting + later scales. This is intentional: once the target is known to be text, + preserving an irrelevant intermediate native-decimal layout would add state + and make valid per-value conversion depend on neighboring rows. +- **Geohashes:** wire precision must equal target bits; no native GEOHASH narrowing is + accepted. `stringColumn` uses `GeoHashes.fromStringTruncatingNl`: empty means + null, parse at most the first 12 base32 characters, require enough bits and + truncate to target precision. Invalid consumed characters fail; characters + beyond the 12-character limit are ignored. The public geohash text setter + first produces a precision-bearing GEOHASH and must not be confused with + `stringColumn`. Geohash-to-text emits bit strings for valid positive wire + precision, not the helper's unreachable negative-precision character branch. +- **Arrays:** DOUBLE_ARRAY copies 8-byte elements; rank must match the known + target, with per-row lengths permitted to vary. The cursor validates rank + 1..32, nonnegative dimensions, checked element count and exact payload size. + Test each Java overload and wrapper, omitted/null/empty arrays, mixed ranks, + ragged inputs and capacity failures. No implicit LONG/DOUBLE element cast or + rank conversion is established by the server. + +Every new pair also needs local rejection/rollback, exact target-wire and +stored-value checks, recovery coverage, and explicit legacy comparisons using +shared vectors. In particular, the fixed-width server cursor treats a null +bitmap as authoritative and otherwise recognizes type-specific sentinels. +BYTE/SHORT/CHAR have no source sentinel; INT minimum, LONG minimum, floating +NaN, IPv4 zero and UUID/LONG256 sentinel limbs can behave differently when a +companion omitted row introduces a bitmap. Test both contexts. The approved +LONG/FLOAT/DOUBLE source-null rules currently specify numeric targets; extending +those rules to missing target families or new input types needs an explicit +contract, not an assumption that every legacy cast has stable null behaviour. + +Decimal cursors are different again: only bitmap bits mark null; raw decimal +sentinel limbs still enter server conversion as supplied values without a bitmap. +Iteration 2.32 characterizes that distinction while preserving public typed- +decimal sentinel/null no-ops. Same/different widths, exact rescaling, explicit +nulls and raw sentinel payloads are covered for decimal targets. For phase 5e, +valid raw native-decimal frames remain a legacy server-formatting check. Public +low-level constructors and `setScale` mutators can also expose coefficients or +scales outside the documented 18/38/76-digit domains. Schema mode does not add +a second validation policy for those values: it uses the same Decimal256 +promotion and unsigned one-byte scale that the server sees on the wire. Public +decimal null sentinels remain setter no-ops, so the server's non-bitmap sentinel +behavior is only a raw-frame boundary. + +### Server boundaries requiring explicit decisions + +These paths are not silently omitted from the inventory and are not newly +approved compatibility exceptions. Characterize them with public tests and +decide the contract before activating the affected Sender paths; copying an +unsafe behaviour merely because it is reachable is not the recommendation. + +1. **BINARY reaches text parsers.** BINARY and VARCHAR share + `QwpStringColumnCursor`. The BOOLEAN, numeric, DATE, timestamp, decimal, + geohash, UUID and LONG256 target branches do not exclude BINARY, so suitable + binary bytes reach those parsers. All those pairs are missing in the binding. + In contrast, CHAR/STRING/VARCHAR/SYMBOL explicitly reject BINARY. Decide + whether binary-to-parser pairs are supported compatibility or a server gap + to close; do not advertise them as normal binary conversions without tests. +2. **LONG_ARRAY is not a supported array conversion.** Auto-creation explicitly + rejects it, despite public client setters and a decoding cursor. For an + existing array column, the append branch accepts either array cursor, but + the rank check runs only for DOUBLE_ARRAY and does not compare element type. + Thus the existing-target route is not proof of correct LONG_ARRAY ingestion + or cross-element conversion. Resolve the server validation/compatibility + gap rather than inventing client-side element casts to cover it. +3. **CHAR-to-text is not uniformly Unicode-safe.** CHAR-to-STRING writes the + UTF-16 code unit; CHAR-to-VARCHAR calls `Utf8StringSink.putAscii(char)`, which + truncates it to one byte. Non-ASCII values can therefore produce corrupted + VARCHAR bytes. A surrogate code unit in STRING also cannot be preserved by + the binding's normal VARCHAR wire encoding. Keep these value-level limits + visible; correct UTF-8 output would require a deliberate compatibility + decision, not an undocumented change disguised as parity. +4. **Designated timestamp fallback is not a conversion catalogue.** Its special + branch can pass a fixed-width cursor to timestamp storage without the + ordinary-column source-type allowlist. Do not derive new public casts from + bit reinterpretation. The client must implement `at`/`atNow` using the pinned + designated unit and the row-completion contract above. +5. **Malformed decimal wire metadata is outside the client contract.** SQL declarations enforce + precision 1..76 and scale 0..precision. The decimal cursor stores a signed + Java scale byte; auto-creation interprets it unsigned and passes it into type + construction whose range checks are assertions. Existing-target conversion + also trusts the unsigned scale. Non-bitmap DECIMAL64/128 sentinel limbs are + formatted as physical minimum integers, while the DECIMAL256 sentinel tuple + formats as an empty non-null string. These are server validation gaps, not + useful schema semantics. Phase 5e nevertheless accepts the physical values + reachable through public low-level decimal constructors and mutators. It + mirrors the current server conversion by promoting to Decimal256 and + interpreting the scale as an unsigned QWP byte; it does not claim that wire + scales 77..255 are valid SQL decimal metadata. Public null sentinels remain + no-ops. Rejecting malformed metadata on the server or hardening the Decimal + API is a separate production change. + +Outside these explicitly listed boundaries, unlisted pairs are not missing +features. In particular, no server conversion supports BOOLEAN-to-SYMBOL, +numeric-to-BOOLEAN, FLOAT/DOUBLE-to-DATE/timestamp, timestamp-to-numeric/DATE/ +SYMBOL, DATE-to-numeric/timestamp, or UUID/LONG256/CHAR/IPv4/geohash/decimal-to- +SYMBOL. SYMBOL does not feed scalar parsers. Neither string-to-IPv4 nor +IPv4-to-integer is accepted: `ipv4Column(text)` parses locally and INT-to-IPv4 +is a distinct migration path. Arrays do not convert to/from scalar values; +decimal and geohash inputs do not convert to ordinary numeric targets. + +### Source-null rules for text output + +Approved on 2026-09-11 for LONG-to-STRING/VARCHAR/SYMBOL only: supplied +`Long.MIN_VALUE` becomes target SQL NULL before formatting, regardless of +surrounding rows. This is an effective write, not a setter no-op; +first-value-wins still applies. Ordinary LONG values use canonical signed +decimal text. Implemented and validated in iteration 2.23 (journal D039). + +Separately approved on 2026-09-11 for UUID-to-STRING/VARCHAR: only the pair with +both limbs `Long.MIN_VALUE` becomes target SQL NULL, as an effective first write. +Every other pair, including either single minimum-valued limb, produces canonical +lowercase UUID text. Implemented and validated in iteration 2.24 (D040). +Native UUID-to-UUID bytes and UUID-to-SYMBOL rejection remain unchanged. + +Separately approved on 2026-09-11 for FLOAT/DOUBLE-to-STRING/VARCHAR/SYMBOL: +every supplied NaN becomes target SQL NULL as an effective first write, +independent of bitmap context. Validate target support and duplicates first. +Finite values, signed zero and infinities use server-equivalent text; FLOAT +widens to DOUBLE before formatting. Legacy behavior remains unchanged. +Implemented and validated in iteration 2.25 (D041). + +Real-server characterization confirms the contrast below for all eleven pairs. +D035 records LONG/UUID evidence; D041 adds the six floating-point pairs: + +| Supplied value | Legacy, no bitmap | Legacy, bitmap marks value present | Schema-mode text rule | +| --- | --- | --- | --- | +| LONG `Long.MIN_VALUE` | NULL | Text `"null"` (not SQL NULL) | NULL (approved) | +| UUID with both limbs `Long.MIN_VALUE` | NULL | `80000000-0000-0000-8000-000000000000` | NULL (approved) | +| FLOAT/DOUBLE NaN | NULL | Text `NaN` (not SQL NULL) | NULL (approved) | + +Other LONG values retain signed decimal formatting; other UUID limb pairs +retain canonical lowercase UUID formatting, including a pair with just one +minimum limb. Legacy-mode bytes and replay remain unchanged. Implementation +must document this deliberate departure from bitmap-present legacy results. +Do not extend the rule to other input families or targets by analogy. + +### Integration work is separate from conversion coverage + +The table does not include missing-table/column inference or Sender lifecycle +integration. Iteration 2.15 implements first-effective-write schema resolution, +designated timestamp row completion and row-boundary adoption. Iteration 2.18 +adds inference for the already-supported setter families, not the missing +families in the conversion inventory. Preserve each public overload's null/no-op +and duplicate rules: +for example decimal/array null and IPv4 text null can be no-ops, binary null +rejects, and a null `Instant` is not a missing timestamp. No fallback to sending +unconverted values, new conversion framework or send-time transformation is +implied by this inventory. + +### Source map + +Line numbers identify this audited working-tree snapshot; method names are the +durable lookup points. + +- [Client Sender][conversion-sender]: public overloads and wire choice, + lines 1118–3017. [Binding][conversion-binding]: implemented setters, + lines 108–403; target selection/parameter rejection, lines 473–545. + [Client buffer][conversion-buffer]: string/binary exception-atomic append, + lines 710–763, 1211–1230 and `restoreStringAppendPositions` (1668). +- [Server cursor selection][conversion-cursors]: `initializeColumnCursor`, + lines 469–562; [fixed-width cursor][conversion-fixed-cursor]: `advanceRow` + and `isCurrentValueSentinelNull`, lines 67–87 and 301–313. +- [Server conversion dispatch and values][conversion-dispatch]: decimal target + entrypoints (367–394), `putDecimalToDecimalColumn` (2087–2109), and decimal + rescaling/range/write helpers (2019–2085, 2433–2522). +- The same [server value-conversion source][conversion-values] implements `putIntegerToNumericColumn` + (1137), integer/float text and symbol writers (790–1041), + `putStringToNumericColumn` (1399), `putStringToTimestampColumn` (1542), + `putStringToLong256Column` (1377), `putCharColumn` (334), + `putBinaryColumn` (207), `putTimestampToStringColumn` (1851) and + `formatFixedOtherValue` (1917). +- The same [value-conversion source][conversion-values] implements decimal + text output (398–516), integer/float input (684–902), string input (1275, + 2282–2429), rescaling/range checks (2019–2279, 2433–2522), geohash input/ + output (1046–1133, 1299–1373), and array copying (`putArrayColumn`, 156). +- [Decimal cursor][conversion-decimal-cursor], lines 73–91 and 196–281; + [array cursor][conversion-array-cursor], lines 179–310; + [geohash cursor][conversion-geo-cursor], lines 122–188; + [GeoHashes][conversion-geohashes], `fromStringTruncatingNl` (238); + [decimal type construction][conversion-column-type], `getDecimalType` (320); + [SQL decimal validation][conversion-sql-parser], lines 269–297. +- [UTF-8 decoding][conversion-utf8], `utf8CharDecode` (1354–1401); + [UTF-8 sink][conversion-utf8-sink], `putAscii(char)` (128–130). + +[conversion-sender]: ../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +[conversion-binding]: ../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +[conversion-buffer]: ../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +[conversion-cursors]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpTableBlockCursor.java +[conversion-fixed-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpFixedWidthColumnCursor.java +[conversion-dispatch]: ../../questdb/core/src/main/java/io/questdb/cairo/wal/WalColumnarRowAppender.java +[conversion-values]: ../../questdb/core/src/main/java/io/questdb/cairo/wal/WalColumnarRowAppender.java +[conversion-decimal-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpDecimalColumnCursor.java +[conversion-array-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpArrayColumnCursor.java +[conversion-geo-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpGeoHashColumnCursor.java +[conversion-geohashes]: ../../questdb/core/src/main/java/io/questdb/cairo/GeoHashes.java +[conversion-column-type]: ../../questdb/core/src/main/java/io/questdb/cairo/ColumnType.java +[conversion-sql-parser]: ../../questdb/core/src/main/java/io/questdb/griffin/SqlParser.java +[conversion-utf8]: ../../questdb/core/src/main/java/io/questdb/std/str/Utf8s.java +[conversion-utf8-sink]: ../../questdb/core/src/main/java/io/questdb/std/str/Utf8StringSink.java + +## Local errors and stale schemas + +Conversions run inside the producer's row-rollback guard. A failed conversion +or lookup cancels the partial row and removes bindings introduced only by that +row; it does not discard completed rows or halt the sender. Diagnostics identify +the table, column, target type, input type and conversion failure. + +Expose these failures through a schema-specific `LineSenderException` subtype +with stable machine-readable reasons. Callers must not parse error messages to +decide recovery. The proposed reason codes and actions are: + +| Reason | Caller action | +|---|---| +| `INVALID_VALUE` | Correct or reject the input; reconstruct the whole row if writing it again. Retrying the same input against the same target will fail. | +| `SCHEMA_CHANGED` | Reconstruct and retry the whole row using the refreshed schema. | +| `SCHEMA_UNAVAILABLE` | Retry the whole row when metadata can be obtained, for example after a timeout or temporary outage. | +| `ACCESS_DENIED` | Resolve authorization; unchanged credentials/permissions do not make an immediate retry useful. | +| `UNSUPPORTED_FEATURE` | Use a supporting endpoint/client for an extension required after upgrade or by pending extended SF, or for an unsupported target type. Do not downgrade or fall back to raw encoding. | + +An old server's absent confirmation before upgrade is normal legacy negotiation, +not `UNSUPPORTED_FEATURE`. + +For failures raised during row construction, the partial row has already been +cancelled. Retrying `flush()` cannot recover it, although a flush may still +publish earlier completed rows. Do not mark these errors as flush-retryable via +the existing `isRetryable()` convention; the schema reason determines whole-row +recovery. Existing transport/flush failures and terminal server NACKs keep their +own delivery and recovery semantics. Test the reason, row outcome and allowed +next public API operation, not only the diagnostic text. + +A cached schema must not reject indefinitely after DDL makes an input valid. +Before reporting a conversion rejection against cached metadata, describe once. +If this operation already obtained a fresh schema, do not look up again. +Freshness belongs to the failing setter, not the whole row: a describe performed +by an earlier setter does not suppress refresh for a later failing setter. + +- If the relevant target is unchanged, preserve the original rejection reason: + `INVALID_VALUE` for an invalid value or `UNSUPPORTED_FEATURE` for an unsupported + conversion. A changed metadata version alone is not `SCHEMA_CHANGED`. +- If the target or table incarnation changed, cancel the row and report + `SCHEMA_CHANGED`. Prepare the fresh snapshot for the next whole-row retry. +- If refresh fails, cancel the partial row and report its actual reason: + temporary unavailability, access denial or unsupported functionality. + +Do not retry only the failing setter using the fresh schema: earlier values +in the same row have already been converted under the old one. The sender +does not retain original inputs for automatic whole-row reconversion. +There is at most one refresh per failing operation, not a DDL retry loop. +An implemented input family rejected against a cached missing column or target +type also gets this refresh, so a newly added or changed server column can be +discovered. An intrinsically unimplemented input family fails directly; looking +up another schema cannot implement that setter. Lookup and authorization errors +are not conversion-retry triggers. + +## Pinned blocks and schema transitions + +Keep two concepts separate: + +- The latest server snapshot, learned through describes or ACKs/NACKs. +- The encoding snapshot pinned to a row and its table block. + +All rows in a schema-mode table block use one encoding snapshot. Its identity +describes the schema used to resolve server-backed bindings, with explicit +inferred bindings for any missing columns. It is not replaced by the latest +cache identity when the block is sent. Wire column definitions still describe +the actual bytes; the identity does not replace those definitions. + +For example, an ACK for an earlier batch may arrive while another block is +being built: + +``` +Server v10: count INT -> current block: v10, INT, four-byte values +Server v11: count LONG -> next block: v11, LONG, eight-byte values +``` + +Publishing v11 cannot relabel existing four-byte values as LONG or start +appending eight-byte values to their column buffer. + +The I/O thread only publishes snapshots. The producer finishes an in-progress +row under its pinned snapshot, or cancels it on error. At the first effective +write of each schema-mode row, including timestamp-only rows, it resolves the +latest cached schema or performs a lookup on a miss. If the identity changed, +it seals completed rows under the old snapshot and installs a new layout before +accepting the new row's first value. This boundary is detected from actual row +state, including rows started without another `table()` call. Repeating +`table()` is not itself a safe transition point. + +For the first implementation, adopting a different identity starts a new +block. Reusing unchanged allocations is allowed, but not relabeling existing +rows. Avoiding splits for provably irrelevant updates is an optional later +optimization, not a reason to mix snapshots. + +Sealing freezes a block; it need not wait for a network send or ACK. Keep sealed +blocks ordered and included in normal pending-row/byte accounting, flush limits +and backpressure. Do not introduce an unbounded queue. Successive generations +of one table can use separate frames; preserve existing multi-table transaction +and deferred-commit boundaries. + +The current `QwpTableBuffer.reset()` preserves column definitions. Transition +therefore needs an explicit producer-owned layout replacement after old rows +have been retained for encoding or encoded. Invalidate column-definition and +designated-timestamp caches with that replacement. Never clear data still +owned by a pending block. + +## Schema feedback through writes + +Each table block in an extended frame carries its name and pinned +`(tableId, metadataVersion)`, or an explicit unknown identity for a missing +table. Multi-table writes carry a separate identity per block. The name remains +the write target, including after drop/recreate; the pair is not a +compare-and-swap precondition or a schema lock. + +The server processes the actual wire values using normal schema resolution and +conversion checks, then compares identities, including changes from auto-create +or added columns. It remains authoritative and may convert the already +converted values again if its schema changed. + +| Outcome | Response | +|---|---| +| Identity matches | Ordinary ACK or NACK | +| Identity differs or is unknown; write succeeds | ACK plus current identity and full schema | +| Identity differs or is unknown; write fails and metadata is available and authorized | NACK plus current identity, full schema and error | + +A version difference alone does not reject data. An old INT block can still +succeed against a new LONG column. Returned metadata must be a coherent +snapshot: identity and column definitions describe the same schema. Updates +are keyed by table name. + +ACKs are cumulative. Accumulate and coalesce pending schema updates until an +eligible response carries them; preserve them across socket backpressure, +deferred frames and commit boundaries. A metadata update never makes otherwise +unacknowledged data safe to acknowledge. + +Order describes and write-response metadata consistently per table name. A +delayed ACK must not overwrite a newer describe, including after drop/recreate. +Do not order different table IDs numerically. Ignore responses from old +connections and cancelled requests. + +For feedback, reserve the top two bits of the response status byte as a mode; +the low six bits retain the existing ACK/NACK status: + +| Mode bits | Payload after the status byte | +|---|---| +| `0x00` | Existing response body, unchanged; no feedback | +| `0x80` | Existing response body, then the complete schema update list | +| `0xC0` | Existing response body only; invalidate all latest-schema cache entries | +| `0x40` | Reserved; reject | + +Both feedback modes require successful schema negotiation. Ordinary cumulative +ACKs and NACKs may carry feedback; durable ACKs retain their existing format +and never carry mode bits. No-feedback responses remain byte-identical even on +negotiated connections. Base response statuses must fit in six bits. + +The update list is `count:u16 LE`, followed by that many entries: +`nameLength:u16 LE`, UTF-8 table name, `schemaPayloadLength:u32 LE`, schema +payload. Count must be positive; names must be valid and unique. Each payload +reuses the SCHEMA control payload without its QWP frame header: +`kind=2:u8`, `requestId=0:i64 LE`, result and any known-schema body. Zero is +reserved for write feedback; DESCRIBE replies still require a positive request +ID. Share schema payload validation, including safe skipping of unknown type +parameters. Column names must satisfy the server's column-name rules and be +unique under its case-insensitive comparison. Validate these once in the shared +schema decoder, before constructing a schema response. Validate the whole +response, exact lengths and trailing-byte exhaustion before publishing metadata +or processing its ACK/NACK outcome. + +Bound the complete `UPDATES` binary response payload to 1 MiB and its WebSocket +header plus payload to the configured response-buffer capacity. If the complete +update set cannot fit, send `INVALIDATE_ALL` instead; never truncate the list, +paginate it or send extra cumulative ACKs to drain metadata. This fallback adds +no bytes to the base ACK/NACK, including at an exact buffer-capacity boundary. +`NONE` and `INVALIDATE_ALL` retain existing base-response size limits: do not +impose the new 1 MiB metadata limit on an otherwise valid large base ACK. If a coherent, +authorized complete update set cannot be returned, invalidation also provides +a bounded fallback without exposing metadata. + +Invalidation affects only the latest-schema cache used for future row starts. +The next use describes the table normally; an individually oversized schema +can then report the existing lookup limit. Pinned snapshots, partial rows, +buffered/persisted bytes and ordinary/durable ACK semantics remain unchanged. + +On the server, retain bounded pending table identities/names rather than stale +encoded snapshots. The current WebSocket path bounds these by its configured +table-writer ceiling, plus at most one failing table name in a rejected group. +Recheck that dependency if table-writer eviction or deferred-group ownership +changes; the response-size limit alone is not a bound on queued names. +Sample coherent metadata and recheck authorization when +serializing each response, in the same serialized response path as DESCRIBE. +Keep committed-prefix ACK feedback separate from feedback for a rejected frame +or deferred group. A NACK may describe authorized tables encountered in that +rejected work; it must not absorb a pending earlier ACK's feedback. Partial +sends retain their exact bytes and feedback ownership until completion. Old +peers must never receive unnegotiated extensions, including during SF replay. + +## Cache and ownership + +Allow up to 1,000,000 latest-schema cache entries per connection, allocated on +demand. No TTL, polling, subscriptions or unsolicited DDL push is required. +Eviction makes the next row's schema resolution a lookup. + +The I/O thread publishes immutable entries in a thread-safe cache; all row, +binding and layout changes are producer-owned. Pinned snapshots may outlive +cache eviction or reconnect. Share immutable metadata where possible and +release it when no longer needed; the entry ceiling is not a total-memory +bound for schemas, converters and pending buffers. + +Use one outstanding lookup because the producer owns row construction; no +general request queue is needed. Its deadline starts at the producer call and +includes waiting for the I/O thread, connection and response. Never reuse a +request ID; correlate completion with the connection that sent it. Timeout, +shutdown and connection loss release waiters. Ignore cancelled or unmatched +replies entirely, including their metadata. + +Validate the fixed control header and request ID before parsing a direct +reply's schema. An identifiable stale reply is ignored even if its schema body +is malformed. Invalid headers that cannot safely identify a reply still fail +the connection. Parse outside the coordinator monitor; if parsing fails, +recheck the existing request identity and deadline before accepting the error +as a current connection failure. Successful parsing retains the final check +before publication. Existing timeout and connection-loss paths remain +responsible for cancellation; response validation must not pre-complete a +request before recycling its connection. + +Cache known schemas and confirmed missing-table results, not denial or +unavailability. A forced refresh removes the previous entry before lookup; +failure does not restore it. Independently received new feedback remains usable. +Existing entries remain usable during an outage; +a later cache miss may wait for reconnection within its own deadline. Installing +a replacement connection clears the latest cache, without changing pinned +snapshots or cancelling a newer request that has not yet been sent. + +A lazy insertion-order cache under the coordinator's monitor is sufficient for +this slice. Normalize names using the server's character comparison rules. +Apply complete validated replies in receive order: server-side metadata +sampling and serialized response sends provide the ordering guarantee above. +Do not add a client wire-sequence barrier: an ACK for older data can legitimately +contain metadata sampled after a newer DESCRIBE. + +## Delivery and remaining limits + +Flush, auto-flush and close encode or publish blocks in their pinned format +without schema lookups or value reconversion. SF persists the original encoded +bytes, frame flag and any pinned identity; replay requires neither original +inputs nor a fresh describe. Extension confirmation is required after upgrade +or when recovering extended SF, but legacy-only recovery before upgrade works +with old servers. No send-time transformation is needed in either mode. A stale +identity can produce repeated schema feedback but is not itself a reason to +reject a replay. Replaying legacy data preserves it; it does not retroactively +provide local conversion validation for those rows. + +Schema feedback is independent of ACK/NACK delivery semantics. A NACK carrying +a new schema still rejects the batch: `SCHEMA_MISMATCH` remains TERMINAL under +[NACK policy v2](qwp-nack-policy-v2.md). Metadata cannot repair persisted bytes, +authorize automatic retry after a terminal error, discard data or advance +ordinary or durable-ACK watermarks. There is no `discard()` API in this feature. + +Concurrent DDL can reject data that was locally valid. It can also make the +server apply another conversion to already converted values. Original inputs +may have lost precision or formatting during the first conversion, so replay +is not equivalent to converting those original inputs against the new schema. +The sender provides neither schema locking nor transparent reconversion after +DDL. Server checks and NACK recovery remain necessary. + +## Simplification opportunities + +These guide integration; the status section records what is implemented. +They do not change the compatibility contract. + +1. **One owner of row data.** Reuse the schema conversion logic with Sender's + existing table buffers. Do not add a parallel buffer lifecycle with separate + row counts, rollback, flush accounting or close handling. Preserve the + distinction between the latest schema and each pending block's pinned + snapshot; reusing storage must never overwrite or relabel pending rows. +2. **One owner of the connection.** Route schema lookups through the existing + I/O loop using a small, bounded request handoff. Reuse its connection, + reconnect and shutdown lifecycle instead of adding another socket owner or + a general-purpose request framework. Keep request correlation, deadlines + and cancellation explicit. The producer still waits for required metadata + before accepting a value and owns conversion; the I/O loop handles the + exchange. +3. **One schema representation.** Share immutable metadata and schema payload + encoding/validation between DESCRIBE and ACK/NACK feedback, while retaining + their distinct message framing and delivery rules. Reuse authorization and + size checks. Keep full snapshots; avoid schema patches, subscriptions or an + additional conversion-version handshake. +4. **A short view of remaining work.** Keep a concise implementation-status + checklist near the beginning of the design, separate from the intended + contract. Distinguish tested components from ordinary Sender activation. + Keep decision history and detailed test evidence in the journal so readers + do not need to reconstruct current status from past iterations. Link to + that evidence rather than maintaining duplicate test logs in the design. +5. **Reuse inferred column definitions.** A confirmed missing target needs no + second schema registry. Let the existing buffer pin the first effective + native type and own rollback. Reuse the existing typed-rejection refresh + when a later row conflicts. Add a guard only for a reachable contract case; + `at()` already finishes a row, so another within-row designated-timestamp + policy is unnecessary. D033 applies this simplification. +6. **Keep local and server rejection tests distinct.** Use public Sender to + verify early errors and row recovery; use explicit legacy frames when the + assertion concerns server conversion or rejection. Do not make a shared + error helper accept either path: that can turn a missing server test into + an apparent pass. Await the relevant ACK when a test depends on feedback, + rather than adding sleeps or accepting multiple error reasons. +7. **Stable negative tests.** For general rejection/rollback checks, use a pair + the server actually rejects, such as LONG-to-BINARY. A temporarily missing + converter such as LONG-to-STRING makes that test obsolete when support is + added. Keep explicit pending-conversion characterization separately. D039 + applies this rule without changing the existing SQL or rollback assertions. +8. **Keep common write paths small.** Isolate an infrequent conversion when + measurements justify it, without adding a conversion framework. In D041, + moving text formatting out of the numeric switch restores native-path + inlining. Shared helper names are not proof of formatter compatibility: + compare actual output before reusing or changing legacy utilities. +9. **Port only the timestamp behavior QWP uses.** The STRING-to-timestamp + converter needs eleven fixed formats, not a general format compiler. Reuse + public JDK timezone metadata, but preserve the server's token ordering, + calendar edge cases and parsed-year recurring-rule calculation. A plain JDK + instant lookup was disproved by an extreme-year input. D043 records the + counterexample and the accepted narrow adapter. Differential and E2E tests + caught a missing fixed-fraction fallback and a retained-leap calendar quirk + before acceptance. No new protocol metadata or parser framework was needed. + Timestamp output likewise uses one fixed formatter and the existing text + sink; it does not need a general calendar-formatting framework (D044). +10. **Reuse the full setter lifecycle.** Activating a conversion includes the + existing typed-error rollback and schema-refresh handling, not just the + binding call. Test refresh through the newly activated setter itself. + D045's CHAR generation test caught a missing catch; adding the standard + catch fixed it without a new abstraction or changed cache policy. +11. **Reuse one decimal conversion tail.** Initialize the existing per-column + scratch value from each source, then share exact rescaling, precision/storage + checks and direct target-width append. LONG therefore needs no separate + decimal encoder, registry or retained state. Its source-null case only locks + the known target scale before marking the bitmap. D049 applies this rule. +12. **Keep one decimal text parser and append tail.** Sync the shared decimal + parser's precision calculation with the current server instead of creating a + schema-only fork, then route the parsed Decimal256 scratch through the same + phase-5a append tail. Enforce exact special-value matching centrally according + to the parser's documented contract, rather than adding target-width branches. + D050 applies this rule. + +These opportunities do not justify a per-setter opt-out, raw-value fallback or +send-time transformation. Partial activation is permitted on the unreleased +development branch to exercise the real public Sender path early. It is not a +release-ready client: missing conversion families still reject some +previously working writes. Preserve legacy-mode behavior and complete the +schema-mode contract before release. + +## Iterative implementation and testing + +Establish a working real-client/real-server E2E test first, then grow the +implementation and its tests together. Do not wait for negotiation, +compatibility and all converters to be implemented before exercising the +complete client-to-server path. Reuse the existing test infrastructure. + +The milestones are: + +1. **E2E baseline.** Write one typed value through the public `Sender` API to + a real server and verify it through SQL. Keep this path working throughout + subsequent changes. +2. **Schema lookup and binding.** Make that write use the server-supplied + schema. Add matching-type, missing-table and lookup-failure cases alongside + the implementation. This slice includes the minimal confirmation gate needed + to use schema mode; milestone 3 covers the broader compatibility matrix. + Land discovery controls/codecs and their real-server protocol tests before + activating the high-level Sender. Until activation, it does not request the + extension and keeps existing behavior. Do not enable schema mode merely to + test negotiation while silently retaining server-side conversions. + Producer-owned conversion components may also land before Sender activation: + exercise real discovery, local conversion/rollback, existing typed QWP bytes + and real-server SQL together. This is conversion-component E2E, not activated + schema mode. Its first increment need not transmit schema identities or + change persisted framing. Introduce the extended frame flag together with + its required negotiation and recovery checks, not as a prerequisite for + testing a local converter. The first public-Sender integration test need not + wait for all conversion families. Start with a known table, implemented inputs + and explicit flush; then extend that same path through row completion, + metadata refresh, schema changes and delivery/recovery. A narrow passing test + does not establish those other paths. No public setter may silently bypass + schema conversion on a supporting connection, even during development. +3. **Negotiation and compatibility.** Add automatic legacy-server compatibility, + mandatory schema mode on supporting servers and the one-way upgrade. Cover + unchanged legacy-client behavior, pending legacy rows and SF during upgrade, + and an old endpoint after upgrade. Test real handshake and framing behavior, + not internal capability fields. +4. **Conversions, one family at a time.** Start with STRING-to-UUID, then add + the other families in separate increments. Each gets valid, invalid, + boundary and null cases, SQL verification and partial-row rollback tests. +5. **Schema evolution.** Add ACK feedback, pinned-block transitions, changes + during row construction and whole-row retry. Test the observable sequence + and resulting data for each behavior as it is introduced. +6. **Recovery scenarios, throughout development.** Extend reconnect, + legacy/extended SF replay, interrupted-delivery and process-restart coverage + whenever the corresponding behavior changes. In particular, recovery tests + start with the first persisted-framing change, not after the other milestones. + +These are milestones, not equally sized iterations or isolated layers. Split +them into small, independently reviewable changes, each production-quality for +its implemented scope. Every iteration includes extensive tests for its new +behavior and failure paths, runs the relevant existing regression suites, and +ends with the real E2E path passing. There is no later "add E2E" or "harden it" +phase. State unfinished scope explicitly; do not release the new client +or advertise the complete extension while its contract is incomplete. + +Tests assert observable contracts: public API results and exceptions, wire +messages, SQL data and documented delivery behavior. No reflection, private +state inspection, test-only production hooks or visibility changes solely for +tests. Unit tests of genuine production component contracts are appropriate; +implementation-specific call counts and internal object layouts are not. + +Use complementary public-Sender tests for integration: a real QuestDB fixture +checks local errors and exact stored rows; a scripted socket peer checks the +negotiated requests, target wire types and pinned identities. SQL alone cannot +prove local conversion because the legacy server can perform the same cast. +A peer omitting schema confirmation proves protocol fallback, not compatibility +with an actual older QuestDB distribution; label that distinction explicitly. + +Use real servers for conversion and ingestion semantics. Scripted protocol +peers and TCP relays may inject malformed responses, hold ACKs, fragment traffic +or disconnect at controlled points; they do not replace real-server E2E tests. +Coordinate concurrent tests through observable events with bounded waits, not +arbitrary sleeps. Seeded fuzzing supplements deterministic cases and must leave +reproducible failures with an independent expected-data model. + +A representative conversion test buffers valid row A, starts row B and supplies +an invalid UUID string to `stringColumn()`, then writes valid row C and flushes. +Assert that the setter throws and SQL contains A and C, with no part of B. +Observe encoded UUID values on the wire where needed to distinguish local +conversion from server-side parsing. Extend the scenario with SF and controlled +recovery as those paths change; do not substitute an assertion that an internal +rollback method ran. + +Process-restart coverage must also create its backlog through public Sender +setters, not only by appending constructed frames to the replay engine. Wait +for local publication and captured data, prove zero server delivery, then force +the producer JVM to stop without closing Sender. A fresh Sender using the same +directory and identity must replay identical QWP payloads and produce exact +rows, including when those payloads contain different pinned schema versions. +Reopening after ACK must not replay acknowledged rows. This tests process death, +not power loss; zero initial delivery is what permits exact row-count assertions +without claiming general exactly-once delivery. + +## Acceptance checks + +- New clients always request the extension. Before upgrade, an old server's + absent confirmation selects existing inferred-type behavior, with unchanged + wire framing, server conversions, missing-value semantics and SF delivery. + Supporting servers enable schema mode automatically; no option disables it. +- Failed handshakes, timeouts and invalid confirmation are not treated as old + servers. Async startup waits for mode selection and cannot bypass a required + first-use lookup. +- Existing clients against new servers see byte-compatible QWP v1 behavior, + with no unsolicited schema messages or extended responses. The base protocol + version remains 1 for both legacy and extended traffic. +- The persisted flag selects table-header parsing. Exercise mixed legacy and + extended SF replay on a negotiated connection, and reject flagged frames on + unnegotiated connections before processing data. +- Upgrade from an old to a supporting server finishes any partial legacy row + unchanged and adopts schema mode at the next row boundary, including without + another `table()` call. Pending legacy blocks and SF remain unchanged and + ordered; new blocks use the extended layout. No send-time conversion occurs. +- Reconnect and background replay re-negotiate support. After upgrade, an old + endpoint receives no pending data and causes neither downgrade nor watermark + advancement. The requirement survives reconnect even with a legacy-only + backlog; recovery detects pending extended SF before sending a legacy prefix. + Legacy-only recovery by a fresh sender remains compatible with old servers. +- The downgrade procedure drains every pending extended slot with a supporting + client under the configured ACK policy. Interrupted or timed-out draining + leaves recoverable data and does not qualify as safe to downgrade. +- Write-only credentials can discover the schema and ingest without SELECT + access. Denied/revoked credentials receive a classified failure and no + unauthorized schema in either DESCRIBE or write-response feedback. +- In schema mode, first use waits for schema. Valid UUID strings and typed UUID + inputs produce the same target representation; malformed strings fail before + row commit/SF. +- Shared conversion vectors cover every input/target pair in the contract, edge + values, nulls, text formatting, timestamp units, decimal rescaling, array + ranks on later rows and geohash precision. Verify final server values too. +- Different input setters can feed one known target column. Inferred missing + columns, duplicate/no-op and name behavior remain covered. +- In schema mode, omitted values and explicit nulls follow target missing-value + semantics, independent of earlier input setter types. Cover boolean-to-STRING omission + becoming NULL, explicit false remaining "false", non-nullable targets and + unchanged results for legacy clients and new clients in legacy mode. +- For LONG input to each numeric target, test `Long.MIN_VALUE` with and without + another omitted value in the same block. Schema mode always emits target + missing/null; legacy mode retains its existing context-dependent behavior. + Duplicate suppression and unsupported-pair rejection still precede normalization. +- For LONG-to-STRING/VARCHAR/SYMBOL, verify canonical decimal bytes, MIN target + NULL versus omission, and MIN-first duplicate suppression. Preserve independent + legacy bitmap-present MIN-to-literal-"null" tests. Distinct/repeated symbols + must retain their own dictionary values when the formatter sink is reused. +- Timestamp overloads test every supported unit's range boundaries, negative + fractions and `Instant` precision at both target resolutions. Include valid + negative instants whose intermediate seconds multiplication would overflow, + duplicate suppression, local failure/rollback and unchanged legacy behavior. +- A stale rejection refreshes once. A changed target cancels the whole row; + retry uses the new snapshot. No setter splices new rules into a partial row. +- Local errors expose the expected reason and recovery action. An invalid row + stays cancelled after flush; whole-row retries after schema refresh or a + recovered lookup can succeed without losing earlier completed rows. Access + denial and unsupported functionality are not mislabeled as transient lookup + failures or flush-retryable errors. +- ACK metadata arriving mid-row leaves that row and completed rows unchanged. + The next adopted snapshot gets a separate block with matching wire types + and identity; exercise INT-to-LONG and cached designated timestamps. +- Transitions work without another `table()` call, preserve pending accounting, + frame order, transaction boundaries, backpressure and completed rows on error. +- Auto-create, multi-table writes, cumulative ACKs, deferred commits, delayed + sends and describe/ACK interleaving preserve schema feedback and watermarks. +- Lookup timeout, offline cache misses, eviction, reconnect and shutdown neither + bypass required resolution nor invalidate pinned data or strand callers. +- Unsupported target types fail explicitly when used; unrelated columns with + unknown types do not block supported writes. The million-entry cache grows + on demand and obsolete pinned metadata is released. +- Flush, close, byte-identical SF replay and terminal-error handling retain their + data semantics. Concurrent DDL can still NACK; metadata never repairs or + silently acknowledges rejected data. From 98be3b0b1bbcf25139ec8a47b5797e59824c4042 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 10:24:11 +0200 Subject: [PATCH 02/51] Raise schema wait budget to 30 seconds The first setter of a row waited at most one second for the initial WebSocket handshake and the schema DESCRIBE round trip, then threw SCHEMA_UNAVAILABLE and dropped the row. One second is a per-row bound on what is really a connection-setup cost, so a cold TLS handshake, a GC pause, or an I/O thread busy draining a backlog could push an ordinary lookup over the line. Every reconnect clears the schema cache, so a flaky link turned into a stream of lost rows. QwpWebSocketSender now reads the budget from a single field defaulting to DEFAULT_SCHEMA_WAIT_MILLIS (30 s). Both the per-row deadline in bindingForEffectiveWrite() and the refresh in refreshAfterSchemaRejection() use it. A healthy lookup still completes in milliseconds; the constant only bounds a dead or still-connecting link. setSchemaWaitMillisForTesting() lets the one test that asserts the deadline fires against an unreachable port keep its short budget. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BDP5ZVN8xijRp9LhQoreMf --- .../cutlass/qwp/client/QwpWebSocketSender.java | 14 ++++++++++++-- .../qwp/client/QwpSchemaSenderIntegrationTest.java | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 91626d3a4..50ec1ccd3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -149,6 +149,10 @@ public class QwpWebSocketSender implements Sender { // sf-client.md section 4.4 floor: drop-oldest under bursts needs a wide // enough window to preserve the trailing category distribution. private static final int MIN_ERROR_INBOX_CAPACITY = 16; + // Upper bound for a row's first setter to wait on the initial handshake + // and a schema DESCRIBE round trip. A healthy lookup takes milliseconds; + // this only bounds a dead or still-connecting link. + private static final long DEFAULT_SCHEMA_WAIT_MILLIS = 30_000L; private static final String WRITE_PATH = "/write/v4"; // Yields the Authorization header value presented on each WebSocket upgrade. A constant for a // fixed token or Basic credential; for an httpTokenProvider it pulls a freshly refreshed token, @@ -394,6 +398,7 @@ public class QwpWebSocketSender implements Sender { // One-way owner capability: shared by foreground and every background // reconnect supplier created by this sender. private volatile boolean schemaRequired; + private long schemaWaitMillis = DEFAULT_SCHEMA_WAIT_MILLIS; // Monotonic per-attempt counter snapshotted onto every connection event // fired from buildAndConnect. Counts every FOREGROUND endpoint try -- // successes and failures alike -- across this sender's lifetime. @@ -2881,6 +2886,11 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } } + @TestOnly + public void setSchemaWaitMillisForTesting(long millis) { + schemaWaitMillis = millis; + } + @TestOnly public void setCursorSendLoopForTesting(CursorWebSocketSendLoop loop) { cursorSendLoop = loop; @@ -4456,7 +4466,7 @@ private QwpSchemaBinding bindingForEffectiveWrite() { schemaResolutionFresh[0] = false; return pinned; } - final long deadlineNanos = System.nanoTime() + 1_000_000_000L; + final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; ensureConnected(); if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(deadlineNanos))) { return null; @@ -4537,7 +4547,7 @@ private RuntimeException refreshAfterSchemaRejection( if (old == null) { return rejection; } - QwpSchemaResponse fresh = cursorSendLoop.refreshSchema(currentTableName, 1_000); + QwpSchemaResponse fresh = cursorSendLoop.refreshSchema(currentTableName, schemaWaitMillis); boolean identityChanged = old.getTableId() != fresh.getTableId() || old.getMetadataVersion() != fresh.getMetadataVersion(); boolean changed = old.getTableId() != fresh.getTableId() diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 3335140f8..2b170b808 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -10,6 +10,7 @@ import io.questdb.client.SenderConnectionEvent; import io.questdb.client.cairo.ColumnType; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.std.Decimal128; @@ -1213,6 +1214,7 @@ public void testFirstNegotiationDeadlineFailsTypedThenRetryUsesAvailableSchema() try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(1_000); sender.table("events"); try { sender.uuidColumn("id", 1, 2); From c1506e219012f9bf385ae148260bc3c54e6323b8 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 11:13:14 +0200 Subject: [PATCH 03/51] feat(qwp): support schema-aware small integers --- .../qwp/client/QwpWebSocketSender.java | 15 +- .../qwp/protocol/QwpSchemaBinding.java | 74 ++++++- .../QwpSchemaSenderIntegrationTest.java | 102 ++++++++++ .../sf/cursor/QwpSchemaReplayNetworkTest.java | 111 +++++++++++ .../qwp/protocol/QwpSchemaBindingTest.java | 187 ++++++++++++++++++ .../cutlass/qwp/small-integer-to-numeric.tsv | 66 +++++++ design/schema-aware-sender-journal.md | 105 ++++++++++ design/schema-aware-sender.md | 133 +++++++++++-- 8 files changed, 765 insertions(+), 28 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 50ec1ccd3..32a316785 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1272,13 +1272,16 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "BYTE"); + binding.byteColumn(columnName, value); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BYTE, false); if (col != null) { col.addByte(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2451,13 +2454,16 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "INT"); + binding.intColumn(columnName, value); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_INT, true); if (col != null) { col.addInt(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -3004,13 +3010,16 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "SHORT"); + binding.shortColumn(columnName, value); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SHORT, false); if (col != null) { col.addShort(value); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 77ff7bd5e..f34fa87df 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -212,6 +212,10 @@ public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { return this; } + public QwpSchemaBinding byteColumn(CharSequence name, byte value) { + return integerNumericColumn(name, value, "BYTE", ColumnType.BYTE, false); + } + public QwpSchemaBinding doubleColumn(CharSequence name, double value) { return floatingNumericColumn(name, value, "DOUBLE"); } @@ -277,6 +281,10 @@ public QwpSchemaBinding floatColumn(CharSequence name, float value) { return floatingNumericColumn(name, value, "FLOAT"); } + public QwpSchemaBinding intColumn(CharSequence name, int value) { + return integerNumericColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); + } + public QwpSchemaBinding longColumn(CharSequence name, long value) { buffer.requireSchemaBinding(this); int index = targetIndex(name, "LONG"); @@ -315,19 +323,19 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { switch (targetType) { case ColumnType.BYTE: if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { - throw invalidRange(name, targetType); + throw invalidRange(name, "LONG", targetType); } column.addByte((byte) value); break; case ColumnType.SHORT: if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { - throw invalidRange(name, targetType); + throw invalidRange(name, "LONG", targetType); } column.addShort((short) value); break; case ColumnType.INT: if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { - throw invalidRange(name, targetType); + throw invalidRange(name, "LONG", targetType); } column.addInt((int) value); break; @@ -355,6 +363,10 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { return this; } + public QwpSchemaBinding shortColumn(CharSequence name, short value) { + return integerNumericColumn(name, value, "SHORT", ColumnType.SHORT, false); + } + public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { buffer.requireSchemaBinding(this); int index = targetIndex(name, "STRING"); @@ -899,8 +911,60 @@ private QwpSchemaBinding decimalNonDecimalColumn( throw unsupported(name, inputType, targetType, "conversion is not implemented"); } - private LineSenderSchemaException invalidRange(CharSequence name, int targetType) { - return error(INVALID_VALUE, name, "LONG", targetType, "value is outside target type range"); + private LineSenderSchemaException invalidRange(CharSequence name, String inputType, int targetType) { + return error(INVALID_VALUE, name, inputType, targetType, "value is outside target type range"); + } + + private QwpSchemaBinding integerNumericColumn( + CharSequence name, + long value, + String inputType, + int inferredType, + boolean sourceNull + ) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, inputType); + int targetType = targetType(index, inferredType); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType)) { + throw unsupported(name, inputType, targetType, "conversion is not implemented"); + } + if (sourceNull) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw invalidRange(name, inputType, targetType); + } + column.addByte((byte) value); + break; + case ColumnType.SHORT: + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw invalidRange(name, inputType, targetType); + } + column.addShort((short) value); + break; + case ColumnType.INT: + column.addInt((int) value); + break; + case ColumnType.LONG: + column.addLong(value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble((double) value); + break; + default: + throw new AssertionError("unsupported numeric target"); + } + return this; } private CharSequence formatLong(long value) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 2b170b808..cf0b1d831 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -48,6 +48,51 @@ public class QwpSchemaSenderIntegrationTest { private static final String DECIMAL_TEXT_CORPUS = "/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv"; + @Test + public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() throws Exception { + assertMemoryLeak(() -> { + int[] targets = { + ColumnType.BYTE, ColumnType.SHORT, ColumnType.INT, + ColumnType.LONG, ColumnType.FLOAT, ColumnType.DOUBLE + }; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(901 + i, 911 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -1).atNow(); + + sender.byteColumn("value", (byte) 99); + try { + switch (i % 3) { + case 0: + sender.byteColumn("failed_b", (byte) 1); + break; + case 1: + sender.shortColumn("failed_b", (short) 1); + break; + default: + sender.intColumn("failed_b", 1); + break; + } + Assert.fail("expected unsupported small-integer-to-UUID conversion"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) 2).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerNumericTable( + "events", 901 + i, 911 + i, numericWireType(target)); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + } + }); + } + @Test public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { assertMemoryLeak(() -> { @@ -1395,6 +1440,25 @@ private static TestWebSocketServer start(TestWebSocketServer server) throws Exce } } + private static byte numericWireType(int targetType) { + switch (targetType) { + case ColumnType.BYTE: + return QwpConstants.TYPE_BYTE; + case ColumnType.SHORT: + return QwpConstants.TYPE_SHORT; + case ColumnType.INT: + return QwpConstants.TYPE_INT; + case ColumnType.LONG: + return QwpConstants.TYPE_LONG; + case ColumnType.FLOAT: + return QwpConstants.TYPE_FLOAT; + case ColumnType.DOUBLE: + return QwpConstants.TYPE_DOUBLE; + default: + throw new AssertionError(ColumnType.nameOf(targetType)); + } + } + private static Sender sender(TestWebSocketServer server) { return Sender.fromConfig("ws::addr=localhost:" + server.getPort() + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" @@ -1757,6 +1821,44 @@ private void nullGeoHashTable(String table, int tableId, long version, int bits) eof(); } + private void smallIntegerNumericTable(String table, int tableId, long version, byte wireType) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + assertSmallIntegerValue(wireType, -1); + assertSmallIntegerValue(wireType, 2); + assertSmallIntegerValue(wireType, 3); + eof(); + } + + private void assertSmallIntegerValue(byte wireType, long value) { + switch (wireType) { + case QwpConstants.TYPE_BYTE: + Assert.assertEquals((byte) value & 0xff, u8()); + break; + case QwpConstants.TYPE_SHORT: + Assert.assertEquals((short) value, in.getShort()); + break; + case QwpConstants.TYPE_INT: + Assert.assertEquals((int) value, in.getInt()); + break; + case QwpConstants.TYPE_LONG: + Assert.assertEquals(value, in.getLong()); + break; + case QwpConstants.TYPE_FLOAT: + Assert.assertEquals(Float.floatToRawIntBits((float) value), in.getInt()); + break; + case QwpConstants.TYPE_DOUBLE: + Assert.assertEquals(Double.doubleToRawLongBits((double) value), in.getLong()); + break; + default: + throw new AssertionError(wireType); + } + } + private void twoGeoHashBlocks(String table, int tableId, long firstVersion, long secondVersion) { messageHeader(2); Assert.assertEquals(0, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java index 365ee32a1..5d4b1d4ab 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; @@ -35,6 +36,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; @@ -124,6 +126,39 @@ public void testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames() th assertNoQuarantine(root); } + @Test + public void testPublicSenderSchemaFrameSurvivesRestartByteForByte() throws Exception { + File root = temp.newFolder("public-sender-restart"); + String slot = new File(root, "default").getAbsolutePath(); + SchemaRecordingHandler firstHandler = new SchemaRecordingHandler(false); + byte[] expected; + try (TestWebSocketServer firstPeer = new TestWebSocketServer(firstHandler)) { + firstPeer.setAdvertiseSchema(true); + firstPeer.start(); + Assert.assertTrue(firstPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(firstPeer.getPort(), root, false))) { + sender.table("small_replay").byteColumn("value", (byte) -7).atNow(); + Assert.assertEquals(0, sender.flushAndGetSequence()); + expected = firstHandler.awaitDataFrame(); + } + } + assertRetained(slot, 0); + + SchemaRecordingHandler replayHandler = new SchemaRecordingHandler(true); + try (TestWebSocketServer replayPeer = new TestWebSocketServer(replayHandler)) { + replayPeer.setAdvertiseSchema(true); + replayPeer.start(); + Assert.assertTrue(replayPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(replayPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertArrayEquals(expected, replayHandler.awaitDataFrame()); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + expected[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + assertNoQuarantine(root); + } + @Test public void testDeferredOnlySchemaTailIsRetainedByOldPeerThenAbortedLocally() throws Exception { File root = temp.newFolder("deferred"); @@ -425,6 +460,82 @@ private static byte[] ok(long sequence) { } } + private static final class SchemaRecordingHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledge; + private final List dataFrames = new ArrayList<>(); + + private SchemaRecordingHandler(boolean acknowledge) { + this.acknowledge = acknowledge; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + sendSchema(client, request.getLong(QwpConstants.HEADER_SIZE + 1)); + return; + } + dataFrames.add(data); + notifyAll(); + if (acknowledge) { + try { + client.sendBinary(RecordingHandler.ok(0)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for schema data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private static void sendSchema(TestWebSocketServer.ClientHandler client, long requestId) { + byte[] value = "value".getBytes(StandardCharsets.UTF_8); + byte[] timestamp = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + Long.BYTES + 1 + Integer.BYTES + Long.BYTES + + Short.BYTES + Short.BYTES + + Short.BYTES + value.length + Integer.BYTES + Short.BYTES + + Short.BYTES + timestamp.length + Integer.BYTES + Short.BYTES; + ByteBuffer response = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + response.putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(301) + .putLong(401) + .putShort((short) 1) + .putShort((short) 2) + .putShort((short) value.length) + .put(value) + .putInt(ColumnType.INT) + .putShort((short) 0) + .putShort((short) timestamp.length) + .put(timestamp) + .putInt(ColumnType.TIMESTAMP_NANO) + .putShort((short) 0); + try { + client.sendBinary(response.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + private static final class IgnoringFactory implements CursorWebSocketSendLoop.ReconnectFactory { private final AtomicInteger attempts = new AtomicInteger(); private final CountDownLatch attemptsLatch; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index 3da239ea0..125124bea 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -521,6 +521,167 @@ public void testLifecycleAndNameValidation() throws Exception { } } + @Test + public void testSmallIntegerNumericConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + try { + long input = Long.parseLong(fields[2]); + int targetType = numericColumnType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", targetType)); + Runnable append = () -> appendSmallInteger(rows, fields[1], input, "n"); + if ("".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + } else { + append.run(); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, + numericWireType(fields[3])); + if ("".equals(fields[4])) { + assertNumericNull(reader, fields[3]); + } else { + Assert.assertEquals(0, reader.byteValue()); + assertNumericValue(reader, fields[3], fields[4]); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(64, count); + } + + @Test + public void testSmallIntegerDuplicateFirstAndFailureRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("a", ColumnType.LONG), + column("only_b", ColumnType.SHORT), + column("bad", ColumnType.BYTE), + column("c", ColumnType.DOUBLE)); + rows.byteColumn("a", (byte) -1).intColumn("a", Integer.MIN_VALUE); + buffer.nextRow(); + + rows.shortColumn("only_b", (short) 5); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.shortColumn("bad", (short) 128)); + rollbackCurrentRow(buffer); + + rows.intColumn("c", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 2, + QwpConstants.TYPE_LONG, QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(2, reader.byteValue()); + Assert.assertEquals(-1, reader.longValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(Double.doubleToRawLongBits(3), reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testIntMinIsNullWithCompanionMissingRowsAcrossNumericTargets() throws Exception { + assertMemoryLeak(() -> { + for (String target : new String[]{"BYTE", "SHORT", "INT", "LONG", "FLOAT", "DOUBLE"}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", numericColumnType(target))); + rows.intColumn("n", Integer.MIN_VALUE); + buffer.nextRow(); + buffer.nextRow(); + rows.intColumn("n", 7); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 3, numericWireType(target)); + Assert.assertEquals(target, 1, reader.byteValue()); + Assert.assertEquals(target, 3, reader.byteValue()); + if ("FLOAT".equals(target)) { + assertNumericValue(reader, target, "0x40e00000"); + } else if ("DOUBLE".equals(target)) { + assertNumericValue(reader, target, "0x401c000000000000"); + } else { + assertNumericValue(reader, target, "7"); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testSmallIntegerMissingSchemaUsesNativeWireTypes() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.byteColumn("b", (byte) -1).shortColumn("s", (short) 2).intColumn("i", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, -1, -1, 1, + QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0xff, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(2, reader.shortValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(3, reader.intValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("uuid", ColumnType.UUID), + column("future", ColumnType.INT, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("uuid", (byte) 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("future", (short) 2)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("uuid", Integer.MIN_VALUE)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("designated", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("designated", 1)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding stale = rows(buffer, column("n", ColumnType.LONG)); + buffer.clear(); + assertIllegalState(() -> stale.byteColumn("n", (byte) 1)); + assertIllegalState(() -> stale.shortColumn("n", (short) 2)); + assertIllegalState(() -> stale.intColumn("n", 3)); + } + } + @Test public void testLongNumericConformanceCorpus() throws Exception { InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( @@ -999,6 +1160,27 @@ private static void assertLongFrame(QwpWebSocketEncoder encoder, int size, long. Assert.assertEquals(size, reader.position()); } + private static void appendSmallInteger( + QwpSchemaBinding rows, + String inputType, + long value, + String column + ) { + switch (inputType) { + case "BYTE": + rows.byteColumn(column, (byte) value); + break; + case "SHORT": + rows.shortColumn(column, (short) value); + break; + case "INT": + rows.intColumn(column, (int) value); + break; + default: + throw new AssertionError(inputType); + } + } + private static void assertNumericValue(Reader reader, String target, String expected) { switch (target) { case "BYTE": @@ -1029,6 +1211,11 @@ private static void assertNumericNull(Reader reader, String target) { Assert.assertEquals(target, 1, reader.byteValue()); } + private static void rollbackCurrentRow(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + private static int numericColumnType(String target) { switch (target) { case "BYTE": return ColumnType.BYTE; diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv new file mode 100644 index 000000000..e429960b1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv @@ -0,0 +1,66 @@ +# case_id input_type input target expected +# FLOAT and DOUBLE expectations are exact raw IEEE-754 bits. INT_MIN is a stable source NULL. +byte_min_byte BYTE -128 BYTE -128 +byte_min_short BYTE -128 SHORT -128 +byte_min_int BYTE -128 INT -128 +byte_min_long BYTE -128 LONG -128 +byte_min_float BYTE -128 FLOAT 0xc3000000 +byte_min_double BYTE -128 DOUBLE 0xc060000000000000 +byte_zero_byte BYTE 0 BYTE 0 +byte_zero_short BYTE 0 SHORT 0 +byte_zero_int BYTE 0 INT 0 +byte_zero_long BYTE 0 LONG 0 +byte_zero_float BYTE 0 FLOAT 0x00000000 +byte_zero_double BYTE 0 DOUBLE 0x0000000000000000 +byte_max_byte BYTE 127 BYTE 127 +byte_max_short BYTE 127 SHORT 127 +byte_max_int BYTE 127 INT 127 +byte_max_long BYTE 127 LONG 127 +byte_max_float BYTE 127 FLOAT 0x42fe0000 +byte_max_double BYTE 127 DOUBLE 0x405fc00000000000 +short_min_byte SHORT -32768 BYTE +short_min_short SHORT -32768 SHORT -32768 +short_min_int SHORT -32768 INT -32768 +short_min_long SHORT -32768 LONG -32768 +short_min_float SHORT -32768 FLOAT 0xc7000000 +short_min_double SHORT -32768 DOUBLE 0xc0e0000000000000 +short_below_byte SHORT -129 BYTE +short_byte_min SHORT -128 BYTE -128 +short_byte_max SHORT 127 BYTE 127 +short_above_byte SHORT 128 BYTE +short_max_byte SHORT 32767 BYTE +short_max_short SHORT 32767 SHORT 32767 +short_max_int SHORT 32767 INT 32767 +short_max_long SHORT 32767 LONG 32767 +short_max_float SHORT 32767 FLOAT 0x46fffe00 +short_max_double SHORT 32767 DOUBLE 0x40dfffc000000000 +int_min_byte INT -2147483648 BYTE +int_min_short INT -2147483648 SHORT +int_min_int INT -2147483648 INT +int_min_long INT -2147483648 LONG +int_min_float INT -2147483648 FLOAT +int_min_double INT -2147483648 DOUBLE +int_min_plus_one_byte INT -2147483647 BYTE +int_min_plus_one_short INT -2147483647 SHORT +int_min_plus_one_int INT -2147483647 INT -2147483647 +int_min_plus_one_long INT -2147483647 LONG -2147483647 +int_min_plus_one_float INT -2147483647 FLOAT 0xcf000000 +int_min_plus_one_double INT -2147483647 DOUBLE 0xc1dfffffffc00000 +int_below_short INT -32769 SHORT +int_short_min INT -32768 SHORT -32768 +int_below_byte INT -129 BYTE +int_byte_min INT -128 BYTE -128 +int_byte_max INT 127 BYTE 127 +int_above_byte INT 128 BYTE +int_128_short INT 128 SHORT 128 +int_short_max INT 32767 SHORT 32767 +int_above_short INT 32768 SHORT +int_32768_int INT 32768 INT 32768 +int_float_round INT 16777217 FLOAT 0x4b800000 +int_double_exact INT 16777217 DOUBLE 0x4170000010000000 +int_max_byte INT 2147483647 BYTE +int_max_short INT 2147483647 SHORT +int_max_int INT 2147483647 INT 2147483647 +int_max_long INT 2147483647 LONG 2147483647 +int_max_float INT 2147483647 FLOAT 0x4f000000 +int_max_double INT 2147483647 DOUBLE 0x41dfffffffc00000 diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 187b7514d..f01dc875d 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -6926,3 +6926,108 @@ production ownership, compatibility wording, exact hashes, raw logs, XML provenance and performance claim bounds. One P3 remains: older source-map line numbers in the design have drifted, although their method names remain correct. That navigation debt is outside phase 5e and does not change the contract. + +## D054 — Post-2.36 simplification and next-iteration plan + +Status: accepted planning decision, 2026-09-14. Revision 56 records the plan; +it does not claim release acceptance or add production code. + +The schema-aware implementation is now committed as client `6fe3ce44` and +server `ffbb7125d2`. Server integration commit `818608c297` pins that exact +client revision. A Maven reactor run using the pinned local client passed the +public-Sender UUID conversion and rollback E2E against the server, so the next +work starts from an integrated baseline rather than another component-only +slice. + +Compatibility becomes a permanent CI and release gate, not another feature +iteration. It must exercise released binaries for new-client/old-server and +old-client/new-server behavior, current new/new schema mode, and the one-way +upgrade failure against an old endpoint. This validates the promised behavior +without adding a compatibility adapter, downgrade encoder or send-time +transformation. + +Timeout productization is removed from the roadmap. The policy is a fixed +30-second deadline with no new public configuration: initial negotiation and +its first describe share one budget, while later cache-miss describes and +refreshes each get their own budget. The timeout change is owned separately +from these conversion slices and is implemented by client commit `98be3b0b`; +it is not recreated or extended by iteration 2.37. + +The next implementation areas are deliberately narrow: iteration 2.37 adds +BYTE/SHORT/INT to the six numeric targets with a small shared helper; iteration +group 2.38 adds fixed-width identity/text support as separate IPv4, LONG256 and +exact-precision GEOHASH slices. Each slice retains a real public-Sender E2E and +the existing wire, SQL, rollback, refresh, recovery, packaging and +affected-performance gates. + +The source audit remains broader than the intended product. BINARY-to-parser +paths, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata are +server-contract questions, not automatic client backlog. The client will not +reproduce parser accidents or validation gaps merely because current server +dispatch can reach them. The encoded 32-bit `ColumnType` remains the single +parameter representation, and no converter registry, schema patch protocol, +retry framework, second request channel or per-path timeout setting is planned. + +Before iteration 2.37 code lands, tests must freeze `Integer.MIN_VALUE` behavior +with and without a companion null bitmap. The recommended schema-mode contract +is stable source NULL, independent of neighboring rows; this remains an explicit +contract checkpoint rather than an assumption copied from LONG. + +## D055 — Small integer inputs use direct numeric target widths + +Status: iteration 2.37 implemented and locally validated, 2026-09-14. Revision +57 is recorded in the client commit containing this decision; the companion +server commit pins that exact client revision. + +BYTE, SHORT and INT setters now support BYTE, SHORT, INT, LONG, FLOAT and DOUBLE +targets. One private helper owns their shared numeric-target check, narrowing +range checks and direct append. The public Sender branches reuse the established +schema-rejection rollback and one-refresh path. Missing columns retain the +setter's native source type. Legacy mode and the established LONG implementation +are unchanged; no registry, converter graph, new buffer state or server +production change was added. + +The source-null decision is now explicit. `Integer.MIN_VALUE` is a stable INT +source null in schema mode, independent of whether another row creates a null +bitmap. BYTE and SHORT have no public source-null sentinel. An unsupported +target still rejects the setter even when the INT value is the null sentinel; +null does not make an otherwise unsupported conversion available. + +Both repositories consume the same 64-case boundary corpus, SHA-256 +`93dd3987145766e6b9d7fff0b592e0a56b67577d83de4de623ed0e8231e3b96d`. +It covers identity, widening, valid and invalid narrowing, FLOAT/DOUBLE raw-bit +rounding and INT nulls. Client component tests assert exact target wire data and +bitmap behavior. Public scripted-Sender tests add partial-row rollback, one +refresh, snapshot pinning and missing-column inference. A public Sender restart +test retains an unacknowledged schema frame and proves byte-identical replay. +The real-server E2E creates all six numeric target tables and verifies the +accepted values through SQL while range failures remain local typed errors. + +Focused client validation passes 91 tests: 35 binding, 47 public Sender and 9 +network replay tests. The established client gate passes 957 tests, followed by +both packaged-JAR integration checks. The focused real-server E2E passes, and +the corrected accumulated server selector passes 379 tests with zero failures, +errors or skips. + +One exploratory server run incorrectly selected every method in +`QwpSenderE2ETest` and `QwpWebSocketSenderReceiverTest`. It ran 631 tests and +reported 3 failures plus 65 errors in conversion families that remain explicitly +unsupported, chiefly arrays, geohashes and parser paths. That run is not a +regression result. A second invalid setup exhausted shared `/tmp`; its generated +512 MiB test directory and crash log were moved under the run-owned server +target directory without touching unrelated temporary files. The valid run uses +project-local temporary storage and only the intentionally supported legacy +methods from the established acceptance selector. + +The affected fixed-work guard runs each identity source for two million warmup +and 50 million measured rows, five times on one CPU. All checksums and work +counts match and hardware counters run at 100%. Average retired instructions are +30.45 billion for BYTE, 30.76 billion for SHORT, 32.54 billion for INT and 35.14 +billion for the established LONG reference. A separate 100-million-row INT run +records no GC after startup. These data reject an obvious helper-path regression; +they are not a before/after speed claim or network/ingestion benchmark because +the prior client had no corresponding schema-aware small-integer operation. + +The fixed 30-second timeout remains outside the conversion roadmap. Iteration +2.37 is rebased on the user's timeout commit `98be3b0b` and does not duplicate +or extend that change. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index f65c881e4..10a97f0e0 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,8 +1,11 @@ # Schema-aware sender: schema-directed encoding -Status: proposal, revision 55. Scope: QWP v1 over WebSocket, with an automatically -negotiated schema extension, legacy-server compatibility and companion server -changes. +Status: implemented on the development branch, revision 57. Scope: QWP v1 over +WebSocket, with an automatically negotiated schema extension, legacy-server +compatibility and companion server changes. The committed baseline contains the +protocol, Sender integration and conversions through iteration 2.36; iteration +2.37 is locally validated. The remaining public-setter conversion contract and +release compatibility gates are not complete. ## Goal and contract @@ -35,11 +38,13 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.36 completes phase 5e native-decimal-to-text -conversion for all three native widths and both text targets. All original 47 -failing conversion cases now pass. -The wider conversion inventory below is not yet complete; this is not release -acceptance. +Current checkpoint: iteration 2.37 adds BYTE, SHORT and INT inputs into all six +numeric targets, including identity. Numeric narrowing is checked locally; +`Integer.MIN_VALUE` is a stable INT source null in schema mode. The committed +baseline remains client `6fe3ce44`, server protocol `ffbb7125d2`, with server +integration commit `818608c297` pinning that client revision. Iteration 2.37 is +validated in this client revision and by its companion server change. The wider +conversion inventory below is not complete; this is not release acceptance. Schema discovery, pinned identity framing, safe replay and ACK/NACK feedback are implemented and tested. Local UUID and LONG-to-numeric conversion now uses @@ -72,6 +77,13 @@ tested: NaN means missing and out-of-range integer conversions fail locally. Coverage includes raw-bit rounding boundaries, legacy differences, rollback and recovered delivery. D025/D026 record the decisions and verification. +BYTE/SHORT/INT input conversion into the six numeric targets is implemented in +iteration 2.37. One narrow helper performs target selection, range checks and +direct target-width append; it does not change the established LONG path. +Shared component, public-Sender, recovered-frame and real-server tests cover 64 +boundary vectors, exact wire/SQL values, rollback, refresh, inference and the +stable INT null contract. D055 records the local acceptance evidence. + STRING input conversion into the six numeric targets is implemented and tested with 429 shared cases, exact wire/stored values, legacy acceptance/rejection, null/sentinel contexts, rollback and recovered delivery. Parsing preserves @@ -98,8 +110,8 @@ The broader server regression selection still has 38 classified failures after the 2.22 expectation/fixture repairs; this is not a release-ready client. Compatibility with a non-confirming socket peer is tested; an actual older QuestDB distribution was not run in this slice. -The integrated lookup deadline is currently fixed at 1000 ms; the proposed -configuration key below is not wired into Sender yet. Complete conversion +The fixed 30-second schema deadline is settled policy and is not part of the +remaining conversion plan. Complete conversion coverage gates release, not the first integrated test. Until that coverage is complete, this is an unreleased development increment: unsupported schema-mode inputs fail explicitly rather than falling back to server conversion. @@ -527,10 +539,14 @@ raw-value encoding. Schema lookup failures never disable schema mode. ## Availability -In schema mode, required lookups have a bounded deadline -(`schema_describe_timeout_millis`, proposed default 1000). Timeout, unavailable -metadata or request saturation throws a lookup error before accepting the -value. It cancels any partial row but does not permanently halt the sender. +In schema mode, required lookups use a fixed 30-second deadline. On the first +write, mode selection and its immediately following describe share one +end-to-end budget; a later cache-miss describe or forced refresh starts its own +30-second budget. This is not a public configuration surface. +Timeout, unavailable metadata or request saturation throws a lookup error +before accepting the value. It cancels any partial row but does not permanently +halt the sender. Do not add automatic retries or separate timeout settings for +the three lookup paths. Known schemas can be used offline; unfamiliar tables require a live lookup. Reconnect invalidates the lookup cache for future row starts, but does not reset @@ -857,8 +873,10 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.36 against the working trees, including uncommitted changes: client base -`981bdb02a471f3b290c89b8e78cbc422610e329e`, server base +refreshed through iteration 2.37 on 2026-09-14. The accepted baseline is committed as +client `6fe3ce44`, server `ffbb7125d2`, with server integration commit +`818608c297` pinning that client revision. The original audit bases were client +`981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; runtime evidence for implemented slices is recorded separately in the journal. Read the actual QWP cursor selection, target dispatch and @@ -887,8 +905,8 @@ non-null conversion. Known compatibility exceptions above still apply. | Public input (source wire type) | Implemented targets | Missing server-accepted targets | | --- | --- | --- | | `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | -| `byteColumn`, `shortColumn` (BYTE, SHORT) | None | Numeric, DATE, timestamps, text, SYMBOL, decimals; includes each input's identity pair | -| `intColumn` (INT) | None | Numeric, DATE, timestamps, text, SYMBOL, decimals, IPv4; includes INT identity | +| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric | DATE, timestamps, text, SYMBOL, decimals | +| `intColumn` (INT) | Numeric | DATE, timestamps, text, SYMBOL, decimals, IPv4 | | `longColumn` (LONG) | Numeric, text, SYMBOL, timestamps, decimals | DATE | | `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | | `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | @@ -1468,13 +1486,19 @@ They do not change the compatibility contract. encoding/validation between DESCRIBE and ACK/NACK feedback, while retaining their distinct message framing and delivery rules. Reuse authorization and size checks. Keep full snapshots; avoid schema patches, subscriptions or an - additional conversion-version handshake. + additional conversion-version handshake. Treat the encoded 32-bit + `ColumnType` as the canonical source of decimal precision/scale, geohash bits + and array element/rank. Keep the existing parameter bytes reserved; do not + add a second per-type parameter model until the encoded type cannot represent + a required contract. 4. **A short view of remaining work.** Keep a concise implementation-status checklist near the beginning of the design, separate from the intended contract. Distinguish tested components from ordinary Sender activation. Keep decision history and detailed test evidence in the journal so readers do not need to reconstruct current status from past iterations. Link to that evidence rather than maintaining duplicate test logs in the design. + Retain method and file names in the source map, but remove brittle source + line numbers when the document is next compacted. 5. **Reuse inferred column definitions.** A confirmed missing target needs no second schema registry. Let the existing buffer pin the first effective native type and own rollback. Reuse the existing typed-rejection refresh @@ -1523,6 +1547,30 @@ They do not change the compatibility contract. phase-5a append tail. Enforce exact special-value matching centrally according to the parser's documented contract, rather than adding target-width branches. D050 applies this rule. +13. **Implement intentional conversions, not every reachable cast.** The server + audit is evidence, not a requirement to reproduce parser accidents or + validation gaps. Keep BINARY-to-parser conversions, LONG_ARRAY, non-ASCII + CHAR-to-VARCHAR and malformed decimal metadata out until their server + contracts are deliberately resolved. Matching a server bug is not + compatibility. +14. **Keep the schema deadline fixed.** Initial negotiation and its first + describe share one 30-second end-to-end budget; later cache-miss describes + and forced refreshes each get their own 30-second budget. Do not add public + configuration, retries, another socket, a request queue or separate timeout + policy per path. +15. **Prefer narrow family helpers to a conversion framework.** A shared helper + for BYTE/SHORT/INT numeric targets is justified because those setters have + the same range-check and append mechanics. Do not introduce a converter + registry or graph. Do not refactor the established LONG hot path without + measurements showing that the change is useful and safe. +16. **Make compatibility a standing gate, not a feature iteration.** Run the + real old/new client-server binary matrix in CI and before release. Each + numbered conversion iteration still keeps a focused current-server E2E test + green; it does not reimplement negotiation or compatibility. +17. **Plan by conversion mechanism.** Group target paths that share a wire + representation, conversion rule and rollback behavior. Do not create one + iteration per matrix cell, and do not combine unrelated server-boundary + decisions merely because they appear in the same source audit. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1538,7 +1586,52 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -The milestones are: +### Plan after iteration 2.37 + +Before the next feature slice, establish the compatibility matrix as a +permanent CI and release gate: + +- New client against the last released old server: no confirmation, unchanged + legacy behavior and no schema frames. +- Last released old client against the new server: unchanged QWP v1 behavior + and no unsolicited schema traffic. +- New client against the new server: schema mode is mandatory and the existing + public-Sender E2E remains green. +- A sender that has confirmed schema support against a later old endpoint: + pending data is retained, with no downgrade or watermark advancement. + +This gate validates existing behavior; it is not a numbered implementation +iteration. Use released binaries for the old sides rather than only scripted +peers. A failure may require a production fix, but the gate itself adds no +protocol mode, adapter or send-time transformation. + +The next implementation slice is: + +1. **Iteration group 2.38 — fixed-width identity and text targets.** Land three + independently reviewable slices: IPv4 to IPv4/STRING/VARCHAR, LONG256 to + LONG256/STRING/VARCHAR, then GEOHASH to the exact same-precision + GEOHASH/STRING/VARCHAR. Each slice gets its own public-Sender real-server E2E, + exact target wire checks, null/omission decisions, rollback, refresh and SF + replay. Preserve geohash bit precision and the existing public overload + semantics; do not use this group to approve unrelated parser paths. + +After these slices, reassess the remaining inventory in this order: integer to +DATE/timestamps; integer to text/SYMBOL/decimals and INT-to-IPv4; +`decimalColumn(CharSequence)`; then DOUBLE arrays after rank metadata is proven +end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII +CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, +not client implementation backlog. + +Every slice keeps the existing observable-contract rules below: public API, +exact wire, SQL result, partial-row rollback and relevant recovery behavior. +Run canonical and packaged-client gates, the server gate against its pinned +client submodule, and affected fixed-work performance checks before acceptance. +Update the journal and exact submodule pin with every accepted slice. + +### Original milestones + +The original milestones below describe how the present architecture was built; +they remain useful as constraints for later work: 1. **E2E baseline.** Write one typed value through the public `Sender` API to a real server and verify it through SQL. Keep this path working throughout From 1310b0dde959c3056bfbdd49723b8ec51dab3cce Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 13:15:26 +0200 Subject: [PATCH 04/51] docs(qwp): record binary compatibility gate --- design/schema-aware-sender-journal.md | 81 ++++++++++++++++++++++++++- design/schema-aware-sender.md | 68 ++++++++++++++-------- 2 files changed, 126 insertions(+), 23 deletions(-) diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index f01dc875d..1dc5798cf 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -1,6 +1,6 @@ # Schema-aware sender implementation journal -North star: [schema-aware sender design](schema-aware-sender.md), revision 55 +North star: [schema-aware sender design](schema-aware-sender.md), revision 58 (revision 9 at kickoff; source-backed clarifications recorded below). This journal records decisions, evidence, review outcomes and remaining work. @@ -7031,3 +7031,82 @@ the prior client had no corresponding schema-aware small-integer operation. The fixed 30-second timeout remains outside the conversion roadmap. Iteration 2.37 is rebased on the user's timeout commit `98be3b0b` and does not duplicate or extend that change. + +## D056 — Released binaries form the standing compatibility gate + +Status: implemented and locally validated, 2026-09-14. Design revision 58. + +Compatibility is now a build gate rather than another conversion iteration. +The gate adds test and CI files only; it does not change client or server +production code, add a protocol adapter, or transform buffered data at send +time. It runs these three process combinations: + +1. current client against QuestDB 10.0.1; +2. client 1.3.9 against the current server; and +3. current client against the current server. + +QuestDB 10.0.1 and client 1.3.9 are the fixed last released baseline before the +schema extension. Their tagged sources contain no schema negotiation headers, +and QuestDB 10.0.1 itself pins client 1.3.9. The peeled source tags are server +`7a391566ac827e0d8c269b91f2415c16b0a32a84` and client +`3d035bdcab9d65b487aa16ffcee2fbd608e84d74`. The gate downloads the +published JARs into its own target directory and verifies SHA-256 +`432d27433836d4430aaca4d701cb0dfac23f7b8281963a5fae97079c0436892d` +for the server and +`41c31bc25646cfe1f045615f53edcfa685a01182749d96c381ea75cd27e7a6a0` +for the client. The shared SLF4J API 2.0.17 dependency is pinned to +`7b751d952061954d5abfed7181c1f645d336091b679891591d63329c622eb832`. +These are deliberate baseline pins, not moving `latest` lookups. + +One Java probe is compiled with `--release 11` against client 1.3.9's public +`Sender` API, then launched in isolated JVMs with exactly one selected client +JAR. It writes one buffered three-row block to a table whose `value` column is +FLOAT: `42`, `Long.MIN_VALUE`, and an omitted value. Explicit auto-flush limits +keep the rows in one block because the last omission creates the bitmap that +makes the old LONG sentinel behavior observable. Legacy mode stores `42.0`, +`-9.223372E18`, and SQL NULL. Schema mode stores `42.0`, SQL NULL, and SQL NULL. +Exact exported CSV distinguishes the modes; mere write success would not. + +The runner starts each server with separate roots and ports, waits for HTTP +readiness, creates WAL tables through the public HTTP endpoint, executes the +probe, polls exported rows for WAL visibility, and retains per-run server logs. +Released artifacts live under `compat/target`, outside the shared Maven cache. +The default source is the canonical Maven Central endpoint; an internal mirror +can be supplied without bypassing checksum validation. + +The same script runs after the existing distribution build in +`.github/workflows/pgwire_stable.yml` and in the Linux release build before +artifacts are copied. It reuses the current server JAR and, when the server pins +a client snapshot, the exact checked-out submodule JAR already built by the job. +When the server pins a released client, it uses that exact resolved dependency +from the local Maven cache. A cold local run, including roughly 40 MiB of +released artifact downloads, passed all three combinations in 5.03 seconds. A +focused client run also passed 60 negotiation, Sender-integration and replay +tests with zero failures, errors or skips. The corresponding focused server run +passed 15 schema discovery, numeric-conversion and identity tests with zero +failures, errors or skips using project-local temporary storage. An initial run +against the already-full shared `/tmp` failed with `No space left`; it is +environmental evidence, not a product regression. + +The process matrix intentionally does not recreate the upgraded-sender state +transition with two server processes. Deterministic socket and replay tests +already prove the stronger exact contract: after schema confirmation, an old +endpoint receives no schema frame, durable and ordinary watermarks do not +advance, and retained bytes replay unchanged to a compatible endpoint. Keeping +that assertion there avoids a timing-sensitive failover rig while the process +gate proves released-binary interoperability and observable mode selection. + +Three independent SOL reviews checked the contract, published artifacts and +CI harness. The first review found an unbounded server `wait`, a probe without +a kill grace period and wording that overstated the public-API level of the +existing watermark tests. The CI review found that Azure's custom Linux +condition replaced its default success gate, that Azure did not retain failure +logs, and that fixed ports plus iteration-counted polling left avoidable false +target and timeout risks. Root replaced those with bounded TERM-to-KILL +shutdown, a probe kill grace period, five-minute CI limits, a success-preserving +Azure condition and failure artifact, occupied-port rejection, real 30-second +deadlines, and accurate socket/replay wording. Released client pins now select +the exact Maven-cache artifact while snapshots select the checked-out submodule +build. A negative run substituting the old client for the current client failed +the schema assertion and cleaned up both servers. Final re-review reports no +blocker or remaining concrete finding. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 10a97f0e0..16ed8d951 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,11 +1,12 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 57. Scope: QWP v1 over +Status: implemented on the development branch, revision 58. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the protocol, Sender integration and conversions through iteration 2.36; iteration -2.37 is locally validated. The remaining public-setter conversion contract and -release compatibility gates are not complete. +2.37 is locally validated. A released-binary compatibility gate is implemented +and locally green. The remaining public-setter conversion contract is not +complete. ## Goal and contract @@ -46,6 +47,15 @@ integration commit `818608c297` pinning that client revision. Iteration 2.37 is validated in this client revision and by its companion server change. The wider conversion inventory below is not complete; this is not release acceptance. +The standing compatibility gate now runs three real process combinations: +current client against QuestDB 10.0.1, client 1.3.9 against the current server, +and current client against the current server. It proves legacy versus schema +encoding from stored values, uses checksum-pinned released artifacts, and runs +in both the stable-client CI workflow and the release pipeline. Exact no-send, +watermark and replay behavior after a confirmed sender reaches an old endpoint +remains covered by deterministic socket and replay tests; it does not need a +second multi-process failover harness. D056 records the gate and local evidence. + Schema discovery, pinned identity framing, safe replay and ACK/NACK feedback are implemented and tested. Local UUID and LONG-to-numeric conversion now uses a non-owning binding over a caller-owned `QwpTableBuffer`; the caller retains @@ -1564,9 +1574,14 @@ They do not change the compatibility contract. registry or graph. Do not refactor the established LONG hot path without measurements showing that the change is useful and safe. 16. **Make compatibility a standing gate, not a feature iteration.** Run the - real old/new client-server binary matrix in CI and before release. Each - numbered conversion iteration still keeps a focused current-server E2E test - green; it does not reimplement negotiation or compatibility. + real old/new client-server binary matrix in CI and before release. Pin the + last pre-extension releases and verify their checksums; do not use moving + `latest` versions. Reuse one public-API probe compiled against the old client + and launch each client in an isolated JVM. Keep exact upgraded-sender + no-send, watermark and replay assertions in the faster deterministic socket + and replay tests instead of adding a flaky process failover rig. Each numbered + conversion iteration still keeps a focused current-server E2E test green; + it does not reimplement negotiation or compatibility. 17. **Plan by conversion mechanism.** Group target paths that share a wire representation, conversion rule and rollback behavior. Do not create one iteration per matrix cell, and do not combine unrelated server-boundary @@ -1588,22 +1603,31 @@ complete client-to-server path. Reuse the existing test infrastructure. ### Plan after iteration 2.37 -Before the next feature slice, establish the compatibility matrix as a -permanent CI and release gate: - -- New client against the last released old server: no confirmation, unchanged - legacy behavior and no schema frames. -- Last released old client against the new server: unchanged QWP v1 behavior - and no unsolicited schema traffic. -- New client against the new server: schema mode is mandatory and the existing - public-Sender E2E remains green. -- A sender that has confirmed schema support against a later old endpoint: - pending data is retained, with no downgrade or watermark advancement. - -This gate validates existing behavior; it is not a numbered implementation -iteration. Use released binaries for the old sides rather than only scripted -peers. A failure may require a production fix, but the gate itself adds no -protocol mode, adapter or send-time transformation. +The permanent compatibility gate is implemented. It launches released and +current artifacts as separate processes and checks these observable contracts: + +- Current client against QuestDB 10.0.1 stores legacy LONG-to-FLOAT conversion. +- Client 1.3.9 against the current server stores the same legacy result. +- Current client against the current server stores the schema-mode result. + +The discriminator is one three-row block written to a FLOAT column: `42`, +`Long.MIN_VALUE`, then a row omitting the value. Legacy mode stores the minimum +long as a float; schema mode treats it as a source null. The old client probe is +compiled against its public API and reused unchanged with each client JAR, so +the gate tests binary compatibility as well as process interoperability. The +released JARs and their SHA-256 hashes are fixed in `versions.env`; current JARs +come from the build under test when the server pins a client snapshot. For a +released client dependency, the gate uses that exact resolved artifact from the +local Maven cache. The same script runs in stable-client CI and before release +artifacts are copied. + +The existing deterministic socket and replay tests remain the authority for exact +negotiation traffic and for a confirmed sender later reaching an old endpoint: +it sends no buffered schema frame, advances no watermark and preserves exact +bytes for replay. Splitting that state transition across two server processes +would add timing and orchestration without improving the contract assertion. +The gate adds no protocol mode, adapter, send-time transformation or production +code. The next implementation slice is: From edf6f34679d1e7e2ef50404f593b881866723364 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 14:25:15 +0200 Subject: [PATCH 05/51] Support schema-aware textual decimals Parse textual decimals through the existing Decimal256 scratch and reuse the established decimal and text conversion paths. Preserve row rollback, schema refresh, duplicate ordering, and missing-column scale semantics without adding retained converter state. Cover exact wire values, every decimal width, text targets, nulls, invalid rows, inference, and real-server ingestion. Record the design decisions, validation evidence, and remaining branch-level blockers. --- .../qwp/client/QwpWebSocketSender.java | 5 +- .../qwp/protocol/QwpSchemaBinding.java | 70 ++++++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 6 +- .../QwpSchemaSenderIntegrationTest.java | 67 ++++++ .../QwpSchemaBindingNativeDecimalTest.java | 208 +++++++++++++++++- design/schema-aware-sender-journal.md | 83 ++++++- design/schema-aware-sender.md | 54 +++-- 7 files changed, 464 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 32a316785..d42f606d1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1736,7 +1736,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(name, "DECIMAL256"); + binding.decimalColumn(name, value, currentDecimal256); return this; } currentDecimal256.ofString(value); @@ -1744,6 +1744,9 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { if (col != null) { col.addDecimal256(currentDecimal256); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index f34fa87df..077a2a5f1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -144,6 +144,58 @@ public QwpSchemaBinding decimalColumn(CharSequence name, Decimal256 value) { return decimalColumn(name, value, "DECIMAL256", QwpConstants.TYPE_DECIMAL256); } + /** + * Parses the public textual-decimal overload after target selection and + * duplicate suppression, then applies the same DECIMAL256 conversion rules + * as the typed overload. The caller owns {@code scratch}; this binding does + * not retain it. + */ + public QwpSchemaBinding decimalColumn(CharSequence name, CharSequence value, Decimal256 scratch) { + buffer.requireSchemaBinding(this); + if (value == null || value.length() == 0) { + return this; + } + int index = targetIndex(name, "DECIMAL256"); + int targetType = targetType(index, ColumnType.DECIMAL256); + if (index < 0) { + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(name); + if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + return this; + } + if (inferred != null && inferred.getType() != QwpConstants.TYPE_DECIMAL256) { + throw unsupported(name, "DECIMAL256", -1, "inferred column type conflict [inferredType=" + + QwpConstants.getTypeName(inferred.getType()) + ']'); + } + parseDecimalText(name, value, scratch, targetType); + inferred = buffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); + if (inferred != null) { + // Keep the legacy public-overload contract: infer DECIMAL256 at + // the parsed natural scale. A parsed special is an effective + // null but does not pin the scale of a later finite value. + inferred.addDecimal256(scratch); + } + return this; + } + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "DECIMAL256", index, targetType); + if (column == null) { + return this; + } + if (!ColumnType.isDecimal(targetType) + && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "DECIMAL256", targetType, "conversion is not implemented"); + } + parseDecimalText(name, value, scratch, targetType); + if (scratch.isNull()) { + if (ColumnType.isDecimal(targetType)) { + column.addSchemaDecimalNull(ColumnType.getDecimalScale(targetType)); + } else { + column.addNull(); + } + return this; + } + return appendDecimal(column, name, scratch, "DECIMAL256", targetType); + } + /** Appends an owned copy of the bytes addressed by {@code slice}. */ public QwpSchemaBinding binaryColumn(CharSequence name, DirectByteSlice slice) { buffer.requireSchemaBinding(this); @@ -869,6 +921,16 @@ private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String if (column == null) { return this; } + return appendDecimal(column, name, value, inputType, targetType); + } + + private QwpSchemaBinding appendDecimal( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + Decimal value, + String inputType, + int targetType + ) { if (!ColumnType.isDecimal(targetType)) { return decimalNonDecimalColumn(column, name, value, inputType, targetType); } @@ -915,6 +977,14 @@ private LineSenderSchemaException invalidRange(CharSequence name, String inputTy return error(INVALID_VALUE, name, inputType, targetType, "value is outside target type range"); } + private void parseDecimalText(CharSequence name, CharSequence value, Decimal256 scratch, int targetType) { + try { + scratch.ofString(value); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "DECIMAL256", targetType, "invalid decimal text"); + } + } + private QwpSchemaBinding integerNumericColumn( CharSequence name, long value, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index f05689c7f..032bef822 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -1495,7 +1495,11 @@ public long getDataAddress() { } public byte getDecimalScale() { - return decimalScale; + // A decimal column containing only nulls has no natural scale yet. + // Scale -1 is internal state, not valid QWP metadata; emit the + // deterministic scale-zero representation without locking it so a + // later finite batch can still select its natural scale. + return decimalScale == -1 ? 0 : decimalScale; } public double[] getDoubleArrayData() { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index cf0b1d831..57dba0bd7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -48,6 +48,59 @@ public class QwpSchemaSenderIntegrationTest { private static final String DECIMAL_TEXT_CORPUS = "/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv"; + @Test + public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 921, 931, ColumnType.getDecimalType(5, 2)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .decimalColumn("value", "1.20m") + .decimalColumn("value", "not-a-decimal") + .atNow(); + + sender.uuidColumn("failed_b", 11, 12); + try { + sender.decimalColumn("value", "not-a-decimal"); + Assert.fail("expected invalid textual decimal"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.decimalColumn("value", "2.5").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimal64Table( + "events", 921, 931, 2, 120, 250); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDecimalTextOverloadFormatsTextAndPreservesNullNoOps() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(941 + targetType, 951, targetType); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", "00123.4500").atNow(); + sender.decimalColumn("value", "1.2300m").atNow(); + sender.decimalColumn("value", "Infinity") + .decimalColumn("value", "not-a-decimal") + .atNow(); + sender.decimalColumn("value", (CharSequence) null) + .decimalColumn("value", "") + .atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).parsedDecimalTextTable( + "events", 941 + targetType, 951); + } + } + }); + } + @Test public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() throws Exception { assertMemoryLeak(() -> { @@ -1743,6 +1796,20 @@ private void decimalTextTable(String table, int tableId, long version, String... eof(); } + private void parsedDecimalTextTable(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0x0c, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals(12, in.getInt()); + Assert.assertEquals("123.451.2300", stringBytes(12)); + eof(); + } + private void twoDecimalBlocks(String table, int tableId, long firstVersion, long secondVersion) { messageHeader(2); Assert.assertEquals(0, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java index 2ee58d1a7..4e9dbc912 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java @@ -84,6 +84,159 @@ public void testCorpusUsesExactTargetDecimalWireAndPreservesCaller() throws Exce }); } + @Test + public void testDecimalTextOverloadUsesEveryExactTargetWidth() throws Exception { + assertMemoryLeak(() -> { + int[] precisions = {2, 4, 9, 18, 38, 76}; + for (int precision : precisions) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", ColumnType.getDecimalType(precision, 1))); + binding.decimalColumn("value", "1.2", new Decimal256()); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, wireTypeForPrecision(precision)); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(12, reader.i64()); + int limbs = precision <= 18 ? 1 : precision <= 38 ? 2 : 4; + for (int i = 1; i < limbs; i++) { + Assert.assertEquals(0, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testDecimalTextOverloadSuppressesDuplicatesAndRollsBackInvalidRows() throws Exception { + assertMemoryLeak(() -> { + int decimal = ColumnType.getDecimalType(5, 1); + Decimal256 scratch = new Decimal256(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", decimal), column("only_b", decimal)); + + binding.decimalColumn("value", "1.2", scratch) + .decimalColumn("value", "not-a-decimal", scratch); + buffer.nextRow(); + + binding.decimalColumn("only_b", "2", scratch); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.decimalColumn("value", "not-a-decimal", scratch)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.decimalColumn("value", "NaN", scratch) + .decimalColumn("value", "still-not-a-decimal", scratch); + buffer.nextRow(); + binding.decimalColumn("value", (CharSequence) null, scratch) + .decimalColumn("value", "", scratch); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 3, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x06, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(12, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadValidatesTargetAfterDuplicateBeforeParsing() throws Exception { + assertMemoryLeak(() -> { + Decimal256 scratch = new Decimal256(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.UUID)); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", "not-a-decimal", scratch)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.uuidColumn("value", 1, 2) + .decimalColumn("value", "not-a-decimal", scratch); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 1, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadMissingInfersDecimal256AtNaturalScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + Decimal256 scratch = new Decimal256(); + binding.decimalColumn("value", "NaN", scratch); + buffer.nextRow(); + binding.decimalColumn("value", "12.3400", scratch); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256, 2); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1234, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadMissingSpecialInfersDecimal256ScaleZeroNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.decimalColumn("value", "-Infinity", new Decimal256()); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.decimalColumn("value", "5.678", new Decimal256()); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = unknownTableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(3, reset.u8()); + Assert.assertEquals(5678, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + @Test public void testDuplicateRollbackResetOmissionAndRetainedTargetScale() throws Exception { assertMemoryLeak(() -> { @@ -221,6 +374,39 @@ public void testLegacyBufferStillChoosesScaleFromFirstValueInEachBatch() throws }); } + @Test + public void testLegacyNullOnlyDecimalUsesWireScaleZeroWithoutLockingNextBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn( + "value", QwpConstants.TYPE_DECIMAL256, true); + column.addDecimal256(Decimal256.NULL_VALUE); + buffer.nextRow(); + + int nullSize = encoder.encode(buffer); + Reader nullOnly = legacyTableReader(encoder, nullSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(1, nullOnly.u8()); + Assert.assertEquals(1, nullOnly.u8()); + Assert.assertEquals(0, nullOnly.u8()); + Assert.assertEquals(nullSize, nullOnly.position()); + + buffer.reset(); + column.addDecimal256(new Decimal256(0, 0, 0, 1234, 4)); + buffer.nextRow(); + int valueSize = encoder.encode(buffer); + Reader value = legacyTableReader(encoder, valueSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(0, value.u8()); + Assert.assertEquals(4, value.u8()); + Assert.assertEquals(1234, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(valueSize, value.position()); + } + }); + } + private static void append(QwpSchemaBinding binding, Object value) { if (value instanceof Decimal64) { binding.decimalColumn("value", (Decimal64) value); @@ -341,12 +527,16 @@ private static QwpSchemaResponse known(int designated, byte[]... columns) { } private static Reader legacyTableReader(QwpWebSocketEncoder encoder, int size) { + return legacyTableReader(encoder, size, QwpConstants.TYPE_DECIMAL64); + } + + private static Reader legacyTableReader(QwpWebSocketEncoder encoder, int size, byte type) { Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.varint()); Assert.assertEquals(1, reader.varint()); - decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reader, "value", type); return reader; } @@ -390,11 +580,15 @@ private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte ty } private static Reader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + return unknownTableReader(encoder, size, type, 1); + } + + private static Reader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(0, reader.u8()); - Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(rows, reader.varint()); Assert.assertEquals(1, reader.varint()); decimalDefinition(reader, "value", type); return reader; @@ -420,6 +614,16 @@ private static int wireLongCount(String target) { return 1; } + private static byte wireTypeForPrecision(int precision) { + if (precision <= 18) { + return QwpConstants.TYPE_DECIMAL64; + } + if (precision <= 38) { + return QwpConstants.TYPE_DECIMAL128; + } + return QwpConstants.TYPE_DECIMAL256; + } + private static long[] wireLimbs(Object value) { if (value instanceof Decimal64) { return new long[]{((Decimal64) value).getValue()}; diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 1dc5798cf..28f38a51b 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -1,13 +1,13 @@ # Schema-aware sender implementation journal -North star: [schema-aware sender design](schema-aware-sender.md), revision 58 +North star: [schema-aware sender design](schema-aware-sender.md), revision 59 (revision 9 at kickoff; source-backed clarifications recorded below). This journal records decisions, evidence, review outcomes and remaining work. The design defines the intended product contract; this journal must not imply that an unimplemented part already works. Corrections are recorded explicitly. -Current status: iterations 1 through 2.36 are accepted for their recorded scopes, including typed +Current status: iterations 1 through 2.37 are accepted for their recorded scopes, including typed microsecond/nanosecond conversion (D020), broader timestamp units and `Instant` with the approved range/precision rules (D021), and text/symbol conversions (D023). Boolean conversion and strict ASCII text-to-boolean parsing are accepted @@ -30,15 +30,16 @@ schema framing and safe SF recovery/replay are implemented and tested. Non-owning bindings provide local UUID, LONG-to-numeric, primitive/`Instant` timestamp, text/symbol, boolean, FLOAT/DOUBLE, STRING numeric, BINARY-target and native decimal-to-decimal, LONG/STRING/FLOAT/DOUBLE-to-decimal and native- -decimal-to-text conversion over caller-owned table buffers. +decimal-to-text conversion over caller-owned table buffers. Iteration 2.38 adds +the textual decimal overload to all decimal and text targets and is locally +validated but not committed. Review corrections are recorded in D016 and D019; the approved LONG null rule is recorded in D013. Automatic row-boundary schema adoption and ordinary Sender conversion are implemented and tested in iteration 2.15. The original 47-method failure ledger is fully resolved after iteration 2.36, but the complete feature is not yet -releasable because the wider conversion inventory and release-level -compatibility validation remain open. +releasable because the wider conversion inventory remains open. D030 records the original 58, D032 the 56-test baseline, D033 the 54-test inference result and D034 the current diagnostic repair. The 2.15 scoped gates passed 410 client tests plus two packaging @@ -7110,3 +7111,75 @@ the exact Maven-cache artifact while snapshots select the checked-out submodule build. A negative run substituting the old client for the current client failed the schema assertion and cleaned up both servers. Final re-review reports no blocker or remaining concrete finding. + +## D057 — Textual decimal input reuses Decimal256 conversion + +Status: iteration 2.38 implemented and locally validated, 2026-09-14. No commit +or parent-repository submodule update has been made. + +The public `decimalColumn(CharSequence)` overload now participates in schema +mode. It remains a DECIMAL256 input, not an alias for `stringColumn`: the Sender +parses once into its existing Decimal256 scratch value, and the binding reuses +the native decimal target conversion and text-formatting tails. Known decimal +targets apply their declared precision and scale. STRING and VARCHAR receive +canonical fixed-point text. This adds no converter registry, retained per-row +object or server production change. + +Target lookup, duplicate suppression and target support checks precede parsing. +An unsupported known target therefore reports `UNSUPPORTED_FEATURE` even for +malformed input, while a duplicate malformed value remains ignored. Java null +and empty input remain no-ops. Parsed `NaN` and infinities remain effective +values that write target NULL; this differs intentionally from typed decimal +null sentinels, which remain no-ops. The public Sender uses the established +partial-row rollback and one-refresh path for local schema errors. + +A confirmed missing column infers DECIMAL256 at the first finite input's natural +scale. A preceding parsed special must not pin scale zero. If a block contains +only decimal nulls, `QwpTableBuffer` emits legal scale-zero wire metadata while +retaining its internal unset scale, so a later finite batch can still choose its +natural scale. This is an encoding-boundary normalization, not new scale state. + +Tests use public behavior and real wire/data results, without reflection or +test-only production hooks. On JDK 25, 114 affected client tests and both +packaged-JAR checks pass with the fork's startup temp root under +`/mnt/pcie5/schema-aware-decimal-text-20260914/client/jdk25-verify`. They cover +all six decimal widths, STRING/VARCHAR formatting, missing inference, parsed +specials, null/empty omission, duplicate ordering, invalid-row rollback, +refresh and exact frames. The companion server selection passes 12 tests for +the new overload plus existing native/string decimal paths under +`/mnt/pcie5/schema-aware-decimal-text-20260914/server/regression`. After +replacing deterministic `returnsOnce()` assertions with the full query battery, +the three new real-server tests pass again under the `server/final3` directory. + +Early server attempts inherited a full shared `/tmp`; server logs proved WAL +mmap failures with `No space left`. No product change followed from those runs. +After the test scratch location was moved to `/mnt/pcie5`, all subsequent test +databases and scratch data used that filesystem. It remains required for future +validation. + +Two broader checks exposed existing branch constraints rather than decimal +failures. First, the Java 8 build stops in the committed, untouched +`QwpSchemaRyuDouble` because it calls Java 9+ `Math.multiplyHigh`; the new code +uses Java 8 syntax and APIs, but a real Java 8 build remains a release gate after +that baseline blocker is repaired. Second, the full client suite blocks in +offline-first tests because the current mandatory initial schema lookup waits +before accepting the first value. The exact recovery failure reproduces at +client baseline `1310b0dd`, and `CloseDrainTest` reaches the same wait on the +current tree. This is the unresolved product conflict between cold offline +buffering, mandatory local schema conversion and the prohibition on send-time +transformation. The simplest coherent contract is to require the first +effective write to establish schema mode; changing that contract needs a +separate decision and is not hidden by excluding individual tests. + +No setter-specific SF replay test was added. This slice changes how the Sender +constructs an ordinary schema-pinned table buffer, not persistence or replay; +the public exact-frame tests cover the newly constructed bytes and the existing +schema-frame replay tests remain byte-oriented. Adding another replay scenario +would duplicate those two contracts without exercising new recovery logic. + +An author and an independent SOL reviewer challenged the first draft. They +found and fixed missing-column dispatch through an unparameterized DECIMAL256 +type and a special-null path that prematurely locked scale zero. Final review +found no implementation blocker, confirmed no apparent per-row heap allocation, +and corrected the affected-test count from an earlier unsubstantiated 218 to +the preserved 114-test reports. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 16ed8d951..e087addd5 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,12 +1,11 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 58. Scope: QWP v1 over +Status: implemented on the development branch, revision 59. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.36; iteration -2.37 is locally validated. A released-binary compatibility gate is implemented -and locally green. The remaining public-setter conversion contract is not -complete. +protocol, Sender integration and conversions through iteration 2.37; iteration +2.38 is locally validated. A released-binary compatibility gate is implemented +and locally green. The remaining public-setter conversion contract is not complete. ## Goal and contract @@ -39,13 +38,13 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.37 adds BYTE, SHORT and INT inputs into all six -numeric targets, including identity. Numeric narrowing is checked locally; -`Integer.MIN_VALUE` is a stable INT source null in schema mode. The committed -baseline remains client `6fe3ce44`, server protocol `ffbb7125d2`, with server -integration commit `818608c297` pinning that client revision. Iteration 2.37 is -validated in this client revision and by its companion server change. The wider -conversion inventory below is not complete; this is not release acceptance. +Current checkpoint: iteration 2.38 adds schema-aware +`decimalColumn(CharSequence)` input for all decimal widths and STRING/VARCHAR +targets. It parses once into the Sender's existing Decimal256 scratch value and +reuses the established decimal append and text-formatting paths. The committed +baseline remains client `1310b0dd` and server `8ad0b92baf`; iteration 2.38 is an +uncommitted, locally validated increment. The wider conversion inventory below +is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: current client against QuestDB 10.0.1, client 1.3.9 against the current server, @@ -108,6 +107,15 @@ append repair also preserves completed rows if string encoding throws midway; successful legacy writes retain their encoding. D029 records the accepted slice, independent reviews and complete verification evidence. +The textual decimal overload now behaves as a DECIMAL256 input in schema mode. +Known decimal targets use their declared precision and scale; STRING/VARCHAR +targets receive canonical fixed-point text. Parsed `NaN` and infinities write +target NULL, while Java `null` and empty input remain no-ops. A missing column +infers DECIMAL256 at the first finite value's natural scale; a null-only block +uses wire scale zero without locking later batches to that scale. The Sender +uses its existing rollback and one-refresh lifecycle, with no new retained +converter state or per-value allocation. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -883,9 +891,9 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.37 on 2026-09-14. The accepted baseline is committed as -client `6fe3ce44`, server `ffbb7125d2`, with server integration commit -`818608c297` pinning that client revision. The original audit bases were client +refreshed through iteration 2.38 on 2026-09-14. The accepted baseline is client +`1310b0dd` and server `8ad0b92baf`, whose submodule pins that exact client +revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; runtime evidence for implemented slices is recorded separately in the journal. @@ -929,7 +937,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `long256Column` (LONG256) | None | LONG256, text | | `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | | `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | -| `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | None | All decimals, text; this is not the `stringColumn` parser path | +| `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | All decimals, text | None; this is not the `stringColumn` parser path | | `geoHashColumn(long, bits)` and `geoHashColumn(CharSequence)` (GEOHASH with source bits) | None | Geohash with exactly matching bit precision, text | | `doubleArray` (all Java ranks and `DoubleArray`; DOUBLE_ARRAY) | None | DOUBLE array of the target rank; identity/shape validation, not element casting | | `longArray` (all Java ranks and `LongArray`; LONG_ARRAY) | None | No supported conversion contract established; see the existing-server gap below | @@ -1586,6 +1594,12 @@ They do not change the compatibility contract. representation, conversion rule and rollback behavior. Do not create one iteration per matrix cell, and do not combine unrelated server-boundary decisions merely because they appear in the same source audit. +18. **Treat textual decimal as an existing source type.** Parse into the + Sender-owned Decimal256 scratch value, then reuse the native decimal append + and text-formatting tails. Keep parsed specials separate only because they + mean an effective target NULL while typed decimal null sentinels are no-ops. + Normalize an unset null-only scale at the encoding boundary; do not add a + second scale state, parser or converter registry. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1601,7 +1615,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.37 +### Plan after iteration 2.38 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1631,7 +1645,7 @@ code. The next implementation slice is: -1. **Iteration group 2.38 — fixed-width identity and text targets.** Land three +1. **Iteration group 2.39 — fixed-width identity and text targets.** Land three independently reviewable slices: IPv4 to IPv4/STRING/VARCHAR, LONG256 to LONG256/STRING/VARCHAR, then GEOHASH to the exact same-precision GEOHASH/STRING/VARCHAR. Each slice gets its own public-Sender real-server E2E, @@ -1641,8 +1655,8 @@ The next implementation slice is: After these slices, reassess the remaining inventory in this order: integer to DATE/timestamps; integer to text/SYMBOL/decimals and INT-to-IPv4; -`decimalColumn(CharSequence)`; then DOUBLE arrays after rank metadata is proven -end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII +then DOUBLE arrays after rank metadata is proven end to end. +BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From 21168defa79c6ae23bc6b86cb3c36cd7b113695a Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 15:20:24 +0200 Subject: [PATCH 06/51] feat(qwp): support schema-aware IPv4 --- .../main/java/io/questdb/client/Sender.java | 12 +- .../qwp/client/QwpWebSocketSender.java | 10 +- .../cutlass/qwp/protocol/QwpConstants.java | 5 + .../qwp/protocol/QwpSchemaBinding.java | 78 ++++++++++ .../java/io/questdb/client/std/Numbers.java | 5 +- .../QwpSchemaSenderIntegrationTest.java | 141 +++++++++++++++++ .../qwp/protocol/QwpConstantsTest.java | 3 + .../qwp/protocol/QwpSchemaBindingTest.java | 145 ++++++++++++++++++ .../questdb/client/test/std/NumbersTest.java | 10 ++ .../client/cutlass/qwp/ipv4-conversions.tsv | 17 ++ design/schema-aware-sender-journal.md | 55 +++++++ design/schema-aware-sender.md | 69 ++++++--- 12 files changed, 521 insertions(+), 29 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 645d7b254..8ae5bbd26 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -653,10 +653,18 @@ default Sender ipv4Column(CharSequence name, int address) { /** * Add an IPv4 column value from a dotted-quad string (e.g. "192.168.1.1"). * + * A null reference skips the column for the current row. The literal + * {@code "null"} (case-insensitive) and the exact address + * {@code "0.0.0.0"} are rejected; omit the setter or pass a null reference + * to write a missing value. + * Legacy dotted aliases such as {@code ".0.0.0.0."} are accepted and + * normalized to the IPv4 NULL value. + * * @param name name of the column - * @param address dotted-quad IPv4 address; must not be null + * @param address dotted-quad IPv4 address, or null to skip the column * @return this instance for method chaining - * @throws LineSenderException if the address fails to parse, or the + * @throws LineSenderException if the address fails to parse, represents a + * rejected NULL sentinel, or the * configured protocol version does not support IPv4 */ default Sender ipv4Column(CharSequence name, CharSequence address) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d42f606d1..a757c0c88 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2494,13 +2494,16 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "IPv4"); + binding.ipv4Column(columnName, address); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_IPv4, true); if (col != null) { col.addIPv4(address); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2545,9 +2548,12 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "IPv4"); + binding.ipv4Column(columnName, address); return this; } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index d4c12a15d..a99f7bf70 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -248,6 +248,7 @@ public static int getFixedTypeSize(byte typeCode) { case TYPE_CHAR: return 2; case TYPE_INT: + case TYPE_IPv4: case TYPE_FLOAT: return 4; case TYPE_LONG: @@ -294,6 +295,9 @@ public static String getTypeName(byte typeCode) { case TYPE_INT: name = "INT"; break; + case TYPE_IPv4: + name = "IPv4"; + break; case TYPE_LONG: name = "LONG"; break; @@ -361,6 +365,7 @@ public static boolean isFixedWidthType(byte typeCode) { (int) typeCode == TYPE_SHORT || (int) typeCode == TYPE_CHAR || (int) typeCode == TYPE_INT || + (int) typeCode == TYPE_IPv4 || (int) typeCode == TYPE_LONG || (int) typeCode == TYPE_FLOAT || (int) typeCode == TYPE_DOUBLE || diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 077a2a5f1..9ac198d60 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -337,6 +337,65 @@ public QwpSchemaBinding intColumn(CharSequence name, int value) { return integerNumericColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); } + public QwpSchemaBinding ipv4Column(CharSequence name, int address) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "IPv4"); + int targetType = targetType(index, ColumnType.IPv4); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "IPv4", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.IPv4 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "IPv4", targetType, "conversion is not implemented"); + } + if (address == Numbers.IPv4_NULL) { + column.addNull(); + } else if (targetType == ColumnType.IPv4) { + column.addIPv4(address); + } else { + column.addString(formatIPv4(address)); + } + return this; + } + + public QwpSchemaBinding ipv4Column(CharSequence name, CharSequence address) { + buffer.requireSchemaBinding(this); + if (address == null) { + return this; + } + int index = targetIndex(name, "IPv4"); + int targetType = targetType(index, ColumnType.IPv4); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "IPv4", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.IPv4 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "IPv4", targetType, "conversion is not implemented"); + } + if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { + throw error(INVALID_VALUE, name, "IPv4", targetType, + "NULL sentinel inputs are rejected; pass a null reference or omit the setter"); + } + final int packed; + try { + packed = Numbers.parseIPv4(address); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "IPv4", targetType, "invalid IPv4 address"); + } + if (packed == Numbers.IPv4_NULL) { + column.addNull(); + } else if (targetType == ColumnType.IPv4) { + column.addIPv4(packed); + } else { + column.addString(formatIPv4(packed)); + } + return this; + } + public QwpSchemaBinding longColumn(CharSequence name, long value) { buffer.requireSchemaBinding(this); int index = targetIndex(name, "LONG"); @@ -859,6 +918,8 @@ private byte wireType(int targetType) { return QwpConstants.TYPE_UUID; case ColumnType.LONG256: return QwpConstants.TYPE_LONG256; + case ColumnType.IPv4: + return QwpConstants.TYPE_IPv4; case ColumnType.BINARY: return QwpConstants.TYPE_BINARY; case ColumnType.TIMESTAMP_MICRO: @@ -1048,6 +1109,23 @@ private CharSequence formatLong(long value) { return sink; } + private CharSequence formatIPv4(int value) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(15); + } else { + sink.clear(); + } + Numbers.append(sink, (value >>> 24) & 0xff); + sink.put('.'); + Numbers.append(sink, (value >>> 16) & 0xff); + sink.put('.'); + Numbers.append(sink, (value >>> 8) & 0xff); + sink.put('.'); + Numbers.append(sink, value & 0xff); + return sink; + } + private CharSequence formatFloating(double value) { StringSink sink = floatingTextSink; if (sink == null) { diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index 78a3bf4c7..5cccdda5b 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -406,7 +406,10 @@ public static int parseIPv4_0(CharSequence sequence, final int p, int lim) throw if (sign == '.') { do { lo++; - } while (sequence.charAt(lo) == '.'); + } while (lo < lim && sequence.charAt(lo) == '.'); + if (lo == lim) { + throw NumericException.instance().put("IPv4 address must have 4 octets, found: 1"); + } } else { throw NumericException.instance().put("invalid IPv4 address: ").put(sequence); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 57dba0bd7..1229f99d2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -843,6 +843,72 @@ public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti }); } + @Test + public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.IPv4, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(211 + target, 221, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .ipv4Column("value", 0) + .ipv4Column("value", "not-an-ip") + .atNow(); + + sender.table("events").ipv4Column("value", 0x05060708); + try { + sender.ipv4Column("failed_b", 0x090a0b0c); + Assert.fail("expected IPv4 to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.uuidColumn("failed_b", 1, 2); + try { + sender.ipv4Column("value", "not-an-ip"); + Assert.fail("expected invalid IPv4 text"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.ipv4Column("value", ".....255.1.2.3......").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).ipv4Table( + "events", 211 + target, 221, target, 0xff010203); + Assert.assertEquals(3, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIpv4GenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(231, 241, ColumnType.IPv4); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").ipv4Column("value", 0x01020304).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 242; + sender.table("events"); + try { + sender.ipv4Column("value", "not-an-ip"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.ipv4Column("value", 0x05060708).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).twoIpv4AndVarcharBlocks("events", 231, 241, 242); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + @Test public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { assertMemoryLeak(() -> { @@ -1132,6 +1198,24 @@ public void testOldPeerUsesLegacyFrameAndNeverDescribes() throws Exception { }); } + @Test + public void testOldPeerUsesLegacyIpv4WireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").ipv4Column("value", 0xc0a80101).atNow(); + sender.table("events").ipv4Column("value", "10.20.30.40").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacyIpv4Table( + "events", "value", 0xc0a80101, 0x0a141e28); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + @Test public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() throws Exception { assertMemoryLeak(() -> { @@ -1674,6 +1758,26 @@ private void legacyVarcharTable(String table, String column, String... values) { eof(); } + private void legacyIpv4Table(String table, String column, int... values) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(values.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_IPv4, u8()); + Assert.assertEquals(0, u8()); + for (int value : values) { + Assert.assertEquals(value, in.getInt()); + } + eof(); + } + private void longTextTable( String table, int tableId, @@ -1710,6 +1814,26 @@ private void longTextTable( eof(); } + private void ipv4Table(String table, int tableId, long version, int targetType, int expectedValue) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = targetType == ColumnType.IPv4 + ? QwpConstants.TYPE_IPv4 + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, tableId, version, 2, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (wireType == QwpConstants.TYPE_IPv4) { + Assert.assertEquals(expectedValue, in.getInt()); + } else { + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(9, in.getInt()); + Assert.assertEquals("255.1.2.3", stringBytes(9)); + } + eof(); + } + private void long256Table(String table, int tableId, long version) { messageHeader(1); Assert.assertEquals(0, varint()); @@ -2179,6 +2303,23 @@ private void twoUuidAndVarcharBlocks(String table, int tableId, long firstVersio eof(); } + private void twoIpv4AndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x01020304, in.getInt()); + + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(7, in.getInt()); + Assert.assertEquals("5.6.7.8", stringBytes(7)); + eof(); + } + private void twoDoubleAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { messageHeader(2); Assert.assertEquals(0, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index ace6d4f15..a4cea0ded 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -46,6 +46,7 @@ public void testGetFixedTypeSize() { Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_SHORT)); Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_CHAR)); Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_INT)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_IPv4)); Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_LONG)); Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_FLOAT)); Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DOUBLE)); @@ -67,6 +68,7 @@ public void testGetFixedTypeSize() { public void testGetTypeName() { Assert.assertEquals("BOOLEAN", QwpConstants.getTypeName(TYPE_BOOLEAN)); Assert.assertEquals("INT", QwpConstants.getTypeName(TYPE_INT)); + Assert.assertEquals("IPv4", QwpConstants.getTypeName(TYPE_IPv4)); Assert.assertEquals("VARCHAR", QwpConstants.getTypeName(TYPE_VARCHAR)); Assert.assertEquals("TIMESTAMP", QwpConstants.getTypeName(TYPE_TIMESTAMP)); Assert.assertEquals("TIMESTAMP_NANOS", QwpConstants.getTypeName(TYPE_TIMESTAMP_NANOS)); @@ -94,6 +96,7 @@ public void testIsFixedWidthType() { Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_SHORT)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_CHAR)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_INT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_IPv4)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_FLOAT)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DOUBLE)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index 125124bea..473f73897 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -1133,6 +1133,151 @@ public void testStringAndTypedUuidEncodeSameLimbs() throws Exception { }); } + @Test + public void testIpv4ConformanceCorpusUsesExactTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_kind\tinput\toutcome\texpected_hex\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + for (int targetType : new int[]{ColumnType.IPv4, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ip", targetType)); + Runnable append = "INT".equals(fields[1]) + ? () -> rows.ipv4Column("ip", Integer.parseInt(fields[2])) + : () -> rows.ipv4Column("ip", fields[2]); + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + continue; + } + append.run(); + buffer.nextRow(); + byte wireType = targetType == ColumnType.IPv4 + ? QwpConstants.TYPE_IPv4 + : QwpConstants.TYPE_VARCHAR; + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.intValue()); + } + } else { + Assert.assertEquals("VALUE", fields[3]); + Assert.assertEquals(0, reader.byteValue()); + if (wireType == QwpConstants.TYPE_IPv4) { + Assert.assertEquals((int) Long.parseUnsignedLong(fields[4], 16), reader.intValue()); + } else { + byte[] expected = fields[5].getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertEquals(fields[5], reader.ascii(expected.length)); + } + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(16, count); + } + + @Test + public void testIpv4DuplicateRollbackNullOmissionAndInference() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("ip", ColumnType.IPv4), column("bad", ColumnType.LONG)); + rows.ipv4Column("ip", 0x01020304).ipv4Column("ip", "not-an-ip"); + buffer.nextRow(); + + rows.ipv4Column("ip", 0x05060708); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("bad", 0x090a0b0c)); + rollbackCurrentRow(buffer); + + rows.ipv4Column("ip", 0); + buffer.nextRow(); + buffer.nextRow(); + rows.ipv4Column("ip", ".0.0.0.0."); + buffer.nextRow(); + rows.ipv4Column("ip", 0x0d0e0f10); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 5, QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0x0e, reader.byteValue()); + Assert.assertEquals(0x01020304, reader.intValue()); + Assert.assertEquals(0x0d0e0f10, reader.intValue()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.ipv4Column("ignored", (CharSequence) null) + .ipv4Column("ip", "192.168.1.1"); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, + QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0xc0a80101, reader.intValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.ipv4Column("ip", 0); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, + QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testIpv4RejectsUnsupportedParameterizedAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("long_value", ColumnType.LONG), + column("parameterized", ColumnType.IPv4, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("long_value", 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("long_value", "not-an-ip")); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("parameterized", "1.2.3.4")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("ts", 1)); + } + } + private static void assertCtorReason(int result, LineSenderSchemaException.Reason reason, String message) { try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index 0467235da..38dd233b0 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -536,6 +536,16 @@ public void testParseIPv4LeadingDots3() { Numbers.parseIPv4("...a.1.2.3.4"); } + @Test(expected = NumericException.class) + public void testParseIPv4OnlyDots() { + Numbers.parseIPv4("...."); + } + + @Test(expected = NumericException.class) + public void testParseIPv4SingleDot() { + Numbers.parseIPv4("."); + } + @Test(expected = NumericException.class) public void testParseIPv4MiddleDots() { Numbers.parseIPv4("1..2..3..4"); diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv new file mode 100644 index 000000000..8044b25f9 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv @@ -0,0 +1,17 @@ +# case_id input_kind input outcome expected_hex expected_text +typed_null INT 0 NULL 00000000 +typed_low INT 1 VALUE 00000001 0.0.0.1 +typed_int_min INT -2147483648 VALUE 80000000 128.0.0.0 +typed_high_bit INT -1062731519 VALUE c0a80101 192.168.1.1 +typed_all_bits INT -1 VALUE ffffffff 255.255.255.255 +text_private TEXT 10.20.30.40 VALUE 0a141e28 10.20.30.40 +text_normalized TEXT .....1.2.3.4...... VALUE 01020304 1.2.3.4 +text_normalized_null TEXT .0.0.0.0. NULL 00000000 +text_null_literal TEXT null INVALID - - +text_null_literal_upper TEXT NULL INVALID - - +text_zero_literal TEXT 0.0.0.0 INVALID - - +text_empty TEXT INVALID - - +text_bad_octet TEXT 1.2.3.256 INVALID - - +text_bad_character TEXT 1.2.3.x INVALID - - +text_single_dot TEXT . INVALID - - +text_only_dots TEXT .... INVALID - - diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 28f38a51b..da2ba75dd 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7183,3 +7183,58 @@ type and a special-null path that prematurely locked scale zero. Final review found no implementation blocker, confirmed no apparent per-row heap allocation, and corrected the affected-test count from an earlier unsubstantiated 218 to the preserved 114-test reports. + +## D058 — IPv4 uses target-native wire data + +Status: iteration 2.39a implemented and locally validated, 2026-09-14. Design +revision 60. The parent-repository test and submodule update land separately. + +Both public `ipv4Column` overloads now participate in schema mode. IPv4 targets +retain the four-byte packed value. STRING and VARCHAR targets receive canonical +dotted-quad text through the binding's existing reusable numeric sink. The +implementation adds no converter registry, address object, parser fork or +per-row string allocation. Legacy mode keeps its existing native IPv4 wire path. + +Packed zero is the public IPv4 null sentinel and becomes an explicit bitmap +NULL for every target. A Java null text reference remains a no-op. The +case-insensitive `"null"` literal and exact `"0.0.0.0"` remain rejected, while +legacy dotted aliases such as `".0.0.0.0."` remain accepted and normalize to +NULL. Target selection and duplicate suppression precede parsing, so an invalid +duplicate stays ignored and an unsupported known target wins over malformed +text. Sender-level schema errors use the established partial-row rollback and +single-refresh lifecycle. + +The implementation review found one pre-existing parser boundary: an address +made only of dots read past the input and leaked `StringIndexOutOfBoundsException` +instead of the public invalid-value error. One bounds check now converts that +case to `NumericException`; `"."` and `"...."` are fixed shared-corpus cases. +The permissive leading/trailing-dot grammar is otherwise unchanged. + +The 16-row corpus covers packed boundaries, canonical and normalized text, +zero/null behavior and malformed input against IPv4, STRING and VARCHAR +targets. Component tests assert exact wire bytes and bitmaps, inference, +omission, duplicate ordering, unsupported/parameterized/designated targets and +rollback. Public scripted-socket tests assert the same target wire plus a schema +change from IPv4 to VARCHAR while the already-buffered block remains native +IPv4. A separate old-peer test proves the integer and text overloads still emit +one unflagged native IPv4 frame without a DESCRIBE. The real-server test sends +all three targets in one batch, verifies exact stored values, local invalid-row +recovery and missing-table inference. A compact store-and-forward test inspects +the persisted frame before replay and verifies the same rows after replay. + +On JDK 25, 238 focused client tests pass with zero failures, errors or skips: +the schema binding, Sender integration, legacy Sender, QWP constants and numeric +parser suites. +Five focused server tests pass against the installed client snapshot: the three +new IPv4 tests plus existing auto-create and schema-aware text-target tests. All +validation used `/mnt/pcie5/schema-aware-ipv4-20260914`. An earlier run inherited +the full shared `/tmp`; logs showed all three WAL writers accepted eight rows, +then STRING and VARCHAR WAL apply suspended on `errno=28`. That run is discarded +as environmental evidence. Setting the forked JVM's `java.io.tmpdir` moved the +JUnit database roots to the project scratch filesystem and the same single-flush +test passed. + +The final SOL review found no production correctness, compatibility, allocation +or complexity blocker. It corrected one design sentence that named the server's +IPv4 formatter as though it existed in the client, and requested the explicit +old-peer setter test above. Re-review found both corrections complete. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index e087addd5..ff739e696 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,11 +1,12 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 59. Scope: QWP v1 over +Status: implemented on the development branch, revision 60. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.37; iteration -2.38 is locally validated. A released-binary compatibility gate is implemented -and locally green. The remaining public-setter conversion contract is not complete. +protocol, Sender integration and conversions through iteration 2.38. Iteration +2.39a (IPv4 identity and text targets) is locally validated. A released-binary +compatibility gate is implemented and locally green. The remaining public-setter +conversion contract is not complete. ## Goal and contract @@ -38,13 +39,10 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.38 adds schema-aware -`decimalColumn(CharSequence)` input for all decimal widths and STRING/VARCHAR -targets. It parses once into the Sender's existing Decimal256 scratch value and -reuses the established decimal append and text-formatting paths. The committed -baseline remains client `1310b0dd` and server `8ad0b92baf`; iteration 2.38 is an -uncommitted, locally validated increment. The wider conversion inventory below -is not complete; this is not release acceptance. +Current checkpoint: iteration 2.39a adds both `ipv4Column` overloads for IPv4, +STRING and VARCHAR targets. Its pre-iteration baseline is client `edf6f346` and +server `fea4d4a6c0`; the IPv4 slice is the current locally validated increment. +The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: current client against QuestDB 10.0.1, client 1.3.9 against the current server, @@ -116,6 +114,15 @@ uses wire scale zero without locking later batches to that scale. The Sender uses its existing rollback and one-refresh lifecycle, with no new retained converter state or per-value allocation. +Both IPv4 overloads now use the server schema in schema mode. An IPv4 target +keeps the four-byte value; STRING and VARCHAR receive canonical dotted-quad +text through the existing reusable numeric sink. Packed zero is written as a +bitmap NULL for every target. A Java null text reference remains a no-op. Exact +`"null"` (case-insensitive) and `"0.0.0.0"` strings remain rejected, while +legacy dotted aliases such as `".0.0.0.0."` remain accepted and normalize to +NULL. Malformed input fails locally with `INVALID_VALUE`, including strings +made only of dots. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -933,7 +940,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `timestampColumn(Instant)` (currently TIMESTAMP micros in legacy Sender) | Timestamps, preserving nanos for the schema-mode nano target; text | None | | `uuidColumn` (UUID) | UUID, text | None | | `charColumn` (CHAR) | CHAR | Text; non-ASCII text output needs the decision below | -| `ipv4Column(int)` and `ipv4Column(CharSequence)` (IPv4; text parsed locally) | None | IPv4, text | +| `ipv4Column(int)` and `ipv4Column(CharSequence)` (IPv4; text parsed locally) | IPv4, text | None | | `long256Column` (LONG256) | None | LONG256, text | | `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | | `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | @@ -948,8 +955,8 @@ binding entrypoint today; record this wire-only coverage without adding a new public API just to mirror the protocol. DATE **targets** from existing integer and string setters are real missing work in the table. -The binding also lacks target representation selection for DATE, -IPv4 and arrays. It rejects extension parameters. +The binding also lacks target representation selection for DATE and arrays. It +rejects extension parameters. Adding a parser alone therefore does not complete these pairs: select the target wire representation and pin its parameters before append. Preserve the existing non-owning binding and buffer ownership. @@ -986,9 +993,11 @@ new casts or copy the unresolved behaviours below. separately. D044 separately approves checked normalization into legacy source precision for text targets, without changing legacy mode. - **Other text output:** UUID uses `Numbers.appendUuid`, LONG256 uses - `Numbers.appendLong256`, IPv4 uses `Numbers.intToIPv4Sink`, DATE uses - `DateFormatUtils.appendDateTime`, and decimal output preserves source scale. - CHAR uses the server's `putAscii` path; see the non-ASCII limitation below. + `Numbers.appendLong256`, DATE uses `DateFormatUtils.appendDateTime`, and + decimal output preserves source scale. The server formats IPv4 through + `Numbers.intToIPv4Sink`; iteration 2.39a uses the client's private reusable + `formatIPv4` sink. CHAR uses the server's `putAscii` path; see the non-ASCII + limitation below. - **Decimals:** all three decimal wire widths reach every decimal target. Decimal input rescaling must be exact: discarded nonzero digits cause precision loss; scale-up/precision/storage overflow fails. Integer input is @@ -1024,6 +1033,13 @@ new casts or copy the unresolved behaviours below. later scales. This is intentional: once the target is known to be text, preserving an irrelevant intermediate native-decimal layout would add state and make valid per-value conversion depend on neighboring rows. +- **IPv4:** the packed zero sentinel becomes a bitmap NULL for IPv4 and text + targets, so its result cannot depend on whether another row creates a bitmap. + Text is parsed locally after target selection and duplicate suppression. + Preserve the legacy dotted grammar: case-insensitive `"null"` and exact + `"0.0.0.0"` are rejected, but dotted aliases that parse to zero normalize to + NULL. Both STRING and VARCHAR receive target-native VARCHAR wire data; no + server-side second conversion is required. - **Geohashes:** wire precision must equal target bits; no native GEOHASH narrowing is accepted. `stringColumn` uses `GeoHashes.fromStringTruncatingNl`: empty means null, parse at most the first 12 base32 characters, require enough bits and @@ -1600,6 +1616,10 @@ They do not change the compatibility contract. mean an effective target NULL while typed decimal null sentinels are no-ops. Normalize an unset null-only scale at the encoding boundary; do not add a second scale state, parser or converter registry. +19. **Keep IPv4 as one primitive path.** Reuse the native four-byte column for + IPv4 targets and the existing reusable numeric sink for canonical text. + Normalize zero to the target bitmap before append. Do not add an address + object, parser fork, converter registry or per-row string allocation. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1615,7 +1635,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.38 +### Plan after iteration 2.39a The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1645,13 +1665,14 @@ code. The next implementation slice is: -1. **Iteration group 2.39 — fixed-width identity and text targets.** Land three - independently reviewable slices: IPv4 to IPv4/STRING/VARCHAR, LONG256 to +1. **Iteration group 2.39 — fixed-width identity and text targets.** IPv4 to + IPv4/STRING/VARCHAR is implemented in 2.39a. Next, land LONG256 to LONG256/STRING/VARCHAR, then GEOHASH to the exact same-precision - GEOHASH/STRING/VARCHAR. Each slice gets its own public-Sender real-server E2E, - exact target wire checks, null/omission decisions, rollback, refresh and SF - replay. Preserve geohash bit precision and the existing public overload - semantics; do not use this group to approve unrelated parser paths. + GEOHASH/STRING/VARCHAR as separate reviewable slices. Each slice gets its own + public-Sender real-server E2E, exact target wire checks, null/omission + decisions, rollback, refresh and SF replay. Preserve geohash bit precision + and the existing public overload semantics; do not use this group to approve + unrelated parser paths. After these slices, reassess the remaining inventory in this order: integer to DATE/timestamps; integer to text/SYMBOL/decimals and INT-to-IPv4; From 0d9663b26f995ad133c093dcf0b0ff39f55f4169 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 16:20:49 +0200 Subject: [PATCH 07/51] feat(qwp): support schema-aware LONG256 --- .../main/java/io/questdb/client/Sender.java | 4 + .../qwp/client/QwpWebSocketSender.java | 12 +- .../qwp/protocol/QwpSchemaBinding.java | 50 ++++++ .../QwpSchemaSenderIntegrationTest.java | 153 ++++++++++++++++++ .../qwp/protocol/QwpSchemaBindingTest.java | 130 +++++++++++++++ .../cutlass/qwp/long256-conversions.tsv | 9 ++ design/schema-aware-sender-journal.md | 88 ++++++++++ design/schema-aware-sender.md | 46 ++++-- 8 files changed, 479 insertions(+), 13 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 8ae5bbd26..56bc4b807 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -674,6 +674,10 @@ default Sender ipv4Column(CharSequence name, CharSequence address) { /** * Add a LONG256 column value, packed as four 64-bit words, least-significant first * (so the 256-bit value is {@code (l3 << 192) | (l2 << 128) | (l1 << 64) | l0}). + * In schema-aware mode, setting all four words to {@link Long#MIN_VALUE} + * writes SQL NULL. Any other combination, including a partial match, is a value. + * Legacy mode preserves its existing behavior: the four words are emitted as + * native LONG256 wire data and server-side null/conversion handling applies. * * @param name name of the column * @param l0 bits 0..63 (least significant) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index a757c0c88..8aeb01e2c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2575,6 +2575,12 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre /** * Adds a LONG256 column value to the current row. + *

      + * In schema-aware mode, setting all four limbs to {@link Long#MIN_VALUE} + * writes SQL NULL. Any other limb combination, including a partial match, + * is a value. Legacy mode preserves its existing behavior: the four limbs + * are emitted as native LONG256 wire data and server-side null/conversion + * handling applies. * * @param columnName the column name * @param l0 the least significant 64 bits @@ -2583,19 +2589,23 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre * @param l3 the most significant 64 bits * @return this sender for method chaining */ + @Override public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "LONG256"); + binding.long256Column(columnName, l0, l1, l2, l3); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG256, true); if (col != null) { col.addLong256(l0, l1, l2, l3); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 9ac198d60..763095a1a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -474,6 +474,30 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { return this; } + public QwpSchemaBinding long256Column(CharSequence name, long l0, long l1, long l2, long l3) { + buffer.requireSchemaBinding(this); + int index = targetIndex(name, "LONG256"); + int targetType = targetType(index, ColumnType.LONG256); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "LONG256", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.LONG256 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw unsupported(name, "LONG256", targetType, "conversion is not implemented"); + } + if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE + && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { + column.addNull(); + } else if (targetType == ColumnType.LONG256) { + column.addLong256(l0, l1, l2, l3); + } else { + column.addString(formatLong256(l0, l1, l2, l3)); + } + return this; + } + public QwpSchemaBinding shortColumn(CharSequence name, short value) { return integerNumericColumn(name, value, "SHORT", ColumnType.SHORT, false); } @@ -1109,6 +1133,32 @@ private CharSequence formatLong(long value) { return sink; } + private CharSequence formatLong256(long l0, long l1, long l2, long l3) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(66); + } else { + sink.clear(); + } + sink.putAscii("0x"); + if (l3 != 0) { + Numbers.appendHex(sink, l3, false); + Numbers.appendHex(sink, l2, true); + Numbers.appendHex(sink, l1, true); + Numbers.appendHex(sink, l0, true); + } else if (l2 != 0) { + Numbers.appendHex(sink, l2, false); + Numbers.appendHex(sink, l1, true); + Numbers.appendHex(sink, l0, true); + } else if (l1 != 0) { + Numbers.appendHex(sink, l1, false); + Numbers.appendHex(sink, l0, true); + } else { + Numbers.appendHex(sink, l0, false); + } + return sink; + } + private CharSequence formatIPv4(int value) { StringSink sink = numericTextSink; if (sink == null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 1229f99d2..5d6d89b97 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -909,6 +909,65 @@ public void testIpv4GenerationRemainsNativeBeforeVarcharRebind() throws Exceptio }); } + @Test + public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.LONG256, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE) + .long256Column("value", 9, 10, 11, 12) + .atNow(); + + sender.table("events").long256Column("value", 5, 6, 7, 8); + try { + sender.long256Column("failed_b", 9, 10, 11, 12); + Assert.fail("expected LONG256 to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.long256Column("value", 1, 2, 3, 4).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).long256TargetTable( + "events", 251 + target, 261, target, 1, 2, 3, 4); + Assert.assertEquals(2, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLong256GenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(281, 291, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").long256Column("value", 1, 2, 3, 4).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 292; + sender.table("events"); + try { + sender.stringColumn("value", "not-long256"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.long256Column("value", 5, 6, 7, 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).twoNativeLong256AndVarcharBlocks( + "events", 281, 291, 292); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + @Test public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { assertMemoryLeak(() -> { @@ -1216,6 +1275,24 @@ public void testOldPeerUsesLegacyIpv4WireAndNeverDescribes() throws Exception { }); } + @Test + public void testOldPeerUsesLegacyLong256WireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").long256Column("value", 1, 2, 3, 4).atNow(); + sender.table("events").long256Column( + "value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacyLong256Table("events", "value"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + @Test public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() throws Exception { assertMemoryLeak(() -> { @@ -1778,6 +1855,29 @@ private void legacyIpv4Table(String table, String column, int... values) { eof(); } + private void legacyLong256Table(String table, String column) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, u8()); + Assert.assertEquals(0, u8()); + for (long value : new long[]{1, 2, 3, 4}) { + Assert.assertEquals(value, i64()); + } + for (int i = 0; i < 4; i++) { + Assert.assertEquals(Long.MIN_VALUE, i64()); + } + eof(); + } + private void longTextTable( String table, int tableId, @@ -1834,6 +1934,39 @@ private void ipv4Table(String table, int tableId, long version, int targetType, eof(); } + private void long256TargetTable( + String table, + int tableId, + long version, + int targetType, + long l0, + long l1, + long l2, + long l3 + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = targetType == ColumnType.LONG256 + ? QwpConstants.TYPE_LONG256 + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, tableId, version, 2, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (wireType == QwpConstants.TYPE_LONG256) { + Assert.assertEquals(l0, i64()); + Assert.assertEquals(l1, i64()); + Assert.assertEquals(l2, i64()); + Assert.assertEquals(l3, i64()); + } else { + String expected = "0x04000000000000000300000000000000020000000000000001"; + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + } + eof(); + } + private void long256Table(String table, int tableId, long version) { messageHeader(1); Assert.assertEquals(0, varint()); @@ -2087,6 +2220,26 @@ private void twoLong256AndVarcharBlocks( eof(); } + private void twoNativeLong256AndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(0, u8()); + for (long value : new long[]{1, 2, 3, 4}) { + Assert.assertEquals(value, i64()); + } + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + String expected = "0x08000000000000000700000000000000060000000000000005"; + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + private void longTimestampTable( String table, int tableId, long version, int targetType, long expectedValue ) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index 473f73897..81cb15baa 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -1278,6 +1278,136 @@ public void testIpv4RejectsUnsupportedParameterizedAndDesignatedTargets() { } } + @Test + public void testLong256ConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/long256-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_l0\tinput_l1\tinput_l2\tinput_l3\toutcome\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + long l0 = Long.parseLong(fields[1]); + long l1 = Long.parseLong(fields[2]); + long l2 = Long.parseLong(fields[3]); + long l3 = Long.parseLong(fields[4]); + for (int targetType : new int[]{ColumnType.LONG256, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + rows.long256Column("value", l0, l1, l2, l3); + buffer.nextRow(); + byte wireType = targetType == ColumnType.LONG256 + ? QwpConstants.TYPE_LONG256 + : QwpConstants.TYPE_VARCHAR; + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, wireType); + if ("NULL".equals(fields[5])) { + Assert.assertEquals("-", fields[6]); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.intValue()); + } + } else { + Assert.assertEquals("VALUE", fields[5]); + Assert.assertEquals(0, reader.byteValue()); + if (wireType == QwpConstants.TYPE_LONG256) { + Assert.assertEquals(l0, reader.longValue()); + Assert.assertEquals(l1, reader.longValue()); + Assert.assertEquals(l2, reader.longValue()); + Assert.assertEquals(l3, reader.longValue()); + } else { + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(fields[6].length(), reader.intValue()); + Assert.assertEquals(fields[6], reader.ascii(fields[6].length())); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(8, count); + }); + } + + @Test + public void testLong256DuplicateNullOmissionRollbackAndMissingInference() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", ColumnType.LONG256), column("bad", ColumnType.UUID)); + rows.long256Column("value", 1, 2, 3, 4) + .long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + buffer.nextRow(); + rows.long256Column("value", 5, 6, 7, 8); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("bad", 9, 10, 11, 12)); + rollbackCurrentRow(buffer); + rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0x06, reader.byteValue()); + Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(3, reader.longValue()); + Assert.assertEquals(4, reader.longValue()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_LONG256, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLong256DuplicatePrecedesUnsupportedAndRejectsParameterizedDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("bad", ColumnType.UUID), + column("parameterized", ColumnType.LONG256, new byte[]{1})); + rows.uuidColumn("bad", 1, 2) + .long256Column("bad", 3, 4, 5, 6); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("bad", 3, 4, 5, 6)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("parameterized", 1, 2, 3, 4)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("ts", 1, 2, 3, 4)); + } + } + private static void assertCtorReason(int result, LineSenderSchemaException.Reason reason, String message) { try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv new file mode 100644 index 000000000..9b45988c2 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv @@ -0,0 +1,9 @@ +# case_id input_l0 input_l1 input_l2 input_l3 outcome expected_text +zero 0 0 0 0 VALUE 0x00 +low_one 1 0 0 0 VALUE 0x01 +second_limb 1 2 0 0 VALUE 0x020000000000000001 +limb_order 1 2 3 4 VALUE 0x04000000000000000300000000000000020000000000000001 +all_ones -1 -1 -1 -1 VALUE 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +partial_min -9223372036854775808 -9223372036854775808 -9223372036854775808 0 VALUE 0x800000000000000080000000000000008000000000000000 +high_sign_bit 0 0 0 -9223372036854775808 VALUE 0x8000000000000000000000000000000000000000000000000000000000000000 +all_min_null -9223372036854775808 -9223372036854775808 -9223372036854775808 -9223372036854775808 NULL - diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index da2ba75dd..f90b1d106 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7238,3 +7238,91 @@ The final SOL review found no production correctness, compatibility, allocation or complexity blocker. It corrected one design sentence that named the server's IPv4 formatter as though it existed in the client, and requested the explicit old-peer setter test above. Re-review found both corrections complete. + +## D059 — LONG256 uses target-native wire data + +Status: iteration 2.39b implemented and locally validated, 2026-09-14. Design +revision 61. The pre-iteration baseline is client `21168def` and server +`d7b94647ae`; neither repository is committed by this decision record. + +The public `long256Column` setter now participates in schema mode. A LONG256 +target receives the four input limbs directly in the existing 32-byte wire +layout. STRING and VARCHAR targets receive direct VARCHAR wire data in the +server's canonical lowercase `0x` form: the most-significant nonzero limb is +written first with even-byte leading width, and each lower limb is padded to 16 +hex digits. The implementation keeps four primitive longs and reuses the +binding's numeric text sink and existing hex appenders. It adds no `BigInteger`, +LONG256 object, converter registry, retained source value, server production +change or per-row string allocation. + +Four `Long.MIN_VALUE` limbs are the source null sentinel in schema mode and are +encoded as an explicit bitmap NULL for LONG256, STRING and VARCHAR targets. Any +partial match remains a value. This removes a legacy row-context dependency: +without a bitmap the server cursor recognizes the four-limb sentinel as null, +but with a bitmap it treats only marked rows as null, so the same unmarked +sentinel converted to text becomes an empty non-null string. Schema mode makes +the intended null explicit before target conversion. Legacy mode deliberately +keeps its native LONG256 wire data and existing server-side behavior. The public +Javadoc states this qualification instead of promising uniform legacy results. + +Target selection and duplicate suppression happen before formatting. Known +unsupported, parameterized and designated targets keep typed rejection; +Sender applies the established partial-row rollback and one-refresh lifecycle. +A confirmed missing column still infers native LONG256. A later schema +generation changes only subsequent buffers: exact scripted-wire coverage proves +an already buffered LONG256 block stays native before a new VARCHAR block is +created. An old peer receives the original unflagged native LONG256 frame and no +DESCRIBE. + +The shared eight-row corpus covers zero, the highest nonzero `l0`, `l1`, `l2` +and `l3` formatter branches, all-one limbs, a partial sentinel, the high sign +bit and the full sentinel null. The client and server resources are +byte-identical with SHA-256 +`a634074af646eb961768599e73727a4a1ab0cd0916dc97a86a8ee4a027723275`. +Component tests assert exact target type, payload and bitmap bytes plus +duplicate, omission, rollback, inference and rejection order. Public socket +tests cover exact schema and legacy frames, rollback and generation rebind. The +real-server corpus uses one Sender and one flush across LONG256, STRING and +VARCHAR tables, with an omitted first value to force late column creation and +two explicit bitmap-null positions. Separate E2E tests cover A/error/C row +recovery, missing-table inference and inspection plus replay of the persisted +target-native frame. A raw legacy fixture pins the bitmap-dependent server +behavior instead of making the schema test silently replace it. + +Final focused validation on JDK 25 passes 96 client tests and nine server tests +with zero failures, errors or skips. The client set is +`QwpSchemaBindingTest,QwpSchemaSenderIntegrationTest`. The server set is +`QwpSchemaLong256E2ETest,QwpSchemaTextSourceNullE2ETest` plus +`QwpSenderE2ETest.testSchemaModePreservesUuidAndLong256NullsForTextTargets`. +An expanded client regression covering the table buffer, encoder, Sender, +constants, prior STRING-to-LONG256 and UUID-to-text bindings, recovered-frame +analysis and schema replay passes 283 tests with zero failures, errors or skips. +All test roots were created under `/mnt/pcie5` by setting `java.io.tmpdir` at +test-JVM startup through `JAVA_TOOL_OPTIONS`. + +An exploratory run of the full `QwpSenderE2ETest` is not green: 137 tests +produced three failures and 41 errors. The LONG256 test changed by this slice +passed. Observed errors include still-missing inventory entries such as +INT-to-IPv4, STRING-to-DATE and LONG_ARRAY, but this run was not fully +classified and is not presented as regression evidence. It confirms the +standing release status: the conversion contract remains incomplete even +though the bounded LONG256 slice is accepted. + +The first server corpus run appeared to return an empty table. The server log +disproved a QWP rejection: LONG256 had applied all eight rows, while the next +STRING WAL apply failed with `errno=28` because shared `/tmp` was full. Passing +`-Djava.io.tmpdir` as an ordinary Maven property did not fix the environment; +`strace -f -yy -e trace=fallocate` showed that Surefire's JVM had cached +`/tmp/junit...` before that property was applied. Supplying it at JVM startup +moved the actual database files to `/mnt/pcie5`, where all three eight-row WAL +applications and queries passed before the final corpus branch was added. The +final nine-row version passes as part of the focused server gate. No production +or test-shape workaround follows from the environmental failure. In particular, +the final test restores the simpler and stronger one-Sender, one-flush shape. + +Independent adversarial review found and closed two issues before acceptance: +the initial Javadoc overstated legacy null semantics, and the seven-row corpus +missed the separate `l1` formatting branch. Re-review found both resolved and no +remaining correctness, compatibility, allocation or complexity issue in this +slice. The wider conversion inventory remains open; the next bounded slice is +native GEOHASH identity and text output with exact target precision. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index ff739e696..054387cdd 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 60. Scope: QWP v1 over +Status: implemented on the development branch, revision 61. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.38. Iteration -2.39a (IPv4 identity and text targets) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.39a. Iteration +2.39b (LONG256 identity and text targets) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,9 +39,9 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.39a adds both `ipv4Column` overloads for IPv4, -STRING and VARCHAR targets. Its pre-iteration baseline is client `edf6f346` and -server `fea4d4a6c0`; the IPv4 slice is the current locally validated increment. +Current checkpoint: iteration 2.39b adds `long256Column` for LONG256, STRING and +VARCHAR targets. Its pre-iteration baseline is client `21168def` and server +`d7b94647ae`; the LONG256 slice is the current locally validated increment. The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: @@ -123,6 +123,16 @@ legacy dotted aliases such as `".0.0.0.0."` remain accepted and normalize to NULL. Malformed input fails locally with `INVALID_VALUE`, including strings made only of dots. +`long256Column` now uses the server schema in schema mode. A LONG256 target +keeps the four limbs in native 32-byte form; STRING and VARCHAR receive the +server's canonical lowercase `0x` text through the reusable numeric sink. Four +`Long.MIN_VALUE` limbs become an explicit bitmap NULL for every supported +target, while any partial match remains a value. Missing columns still infer +LONG256. Legacy mode keeps its existing native wire data and server-side +conversion behavior. Shared client/server vectors, exact wire checks, public +Sender ingestion, rollback, schema rebind, legacy framing and persisted-frame +replay cover the slice. D059 records the decision and evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -941,7 +951,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `uuidColumn` (UUID) | UUID, text | None | | `charColumn` (CHAR) | CHAR | Text; non-ASCII text output needs the decision below | | `ipv4Column(int)` and `ipv4Column(CharSequence)` (IPv4; text parsed locally) | IPv4, text | None | -| `long256Column` (LONG256) | None | LONG256, text | +| `long256Column` (LONG256) | LONG256, text | None | | `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | | `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | | `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | All decimals, text | None; this is not the `stringColumn` parser path | @@ -1040,6 +1050,13 @@ new casts or copy the unresolved behaviours below. `"0.0.0.0"` are rejected, but dotted aliases that parse to zero normalize to NULL. Both STRING and VARCHAR receive target-native VARCHAR wire data; no server-side second conversion is required. +- **LONG256:** four `Long.MIN_VALUE` limbs are the source null sentinel in + schema mode and become an explicit bitmap NULL for LONG256 and text targets; + a partial match remains a value. LONG256 keeps the four little-endian limbs. + STRING and VARCHAR receive direct VARCHAR wire data formatted as lowercase + `0x` text with the most-significant nonzero limb first, even-byte leading + width and 16 hex digits for each lower limb. Legacy native wire behavior is + unchanged, including its bitmap-dependent server-side text conversion. - **Geohashes:** wire precision must equal target bits; no native GEOHASH narrowing is accepted. `stringColumn` uses `GeoHashes.fromStringTruncatingNl`: empty means null, parse at most the first 12 base32 characters, require enough bits and @@ -1620,6 +1637,11 @@ They do not change the compatibility contract. IPv4 targets and the existing reusable numeric sink for canonical text. Normalize zero to the target bitmap before append. Do not add an address object, parser fork, converter registry or per-row string allocation. +20. **Keep LONG256 as four primitive limbs.** Reuse the native 32-byte column + for identity and the existing numeric sink plus hex appenders for text. + Normalize the four-limb sentinel to the target bitmap before append. Do not + add `BigInteger`, a LONG256 object, a formatter registry, retained source + values or per-row string allocation. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1635,7 +1657,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.39a +### Plan after iteration 2.39b The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1666,10 +1688,10 @@ code. The next implementation slice is: 1. **Iteration group 2.39 — fixed-width identity and text targets.** IPv4 to - IPv4/STRING/VARCHAR is implemented in 2.39a. Next, land LONG256 to - LONG256/STRING/VARCHAR, then GEOHASH to the exact same-precision - GEOHASH/STRING/VARCHAR as separate reviewable slices. Each slice gets its own - public-Sender real-server E2E, exact target wire checks, null/omission + IPv4/STRING/VARCHAR is implemented in 2.39a, and LONG256 to + LONG256/STRING/VARCHAR is implemented in 2.39b. Next, land GEOHASH to the + exact same-precision GEOHASH/STRING/VARCHAR as a separate reviewable slice. + It gets a public-Sender real-server E2E, exact target wire checks, null/omission decisions, rollback, refresh and SF replay. Preserve geohash bit precision and the existing public overload semantics; do not use this group to approve unrelated parser paths. From 8932076f2418186e480c5de454532a6087a9bd63 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 17:00:37 +0200 Subject: [PATCH 08/51] feat(qwp): support schema-aware GEOHASH --- .../main/java/io/questdb/client/Sender.java | 8 + .../qwp/client/QwpWebSocketSender.java | 10 +- .../qwp/protocol/QwpSchemaBinding.java | 65 ++++++ .../QwpSchemaSenderIntegrationTest.java | 157 ++++++++++++++ .../qwp/protocol/QwpSchemaBindingTest.java | 200 ++++++++++++++++++ .../qwp/native-geohash-conversions.tsv | 16 ++ design/schema-aware-sender-journal.md | 65 ++++++ design/schema-aware-sender.md | 76 ++++--- 8 files changed, 568 insertions(+), 29 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 56bc4b807..eaeb4d005 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -570,6 +570,10 @@ default long flushAndGetSequence() { * Precision is locked the first time a value is added to the column: subsequent * rows must use the same precision or a {@link LineSenderException} is thrown. * To mark the value NULL, do not call this method for the current row. + * In schema-aware mode, the server column must be a GEOHASH with exactly the + * same precision, STRING, or VARCHAR. Text targets receive a binary string + * containing exactly {@code precisionBits} characters. Legacy mode keeps the + * native GEOHASH wire representation and server-side conversion behavior. * * @param name name of the column * @param bits packed geohash; low {@code precisionBits} bits significant @@ -595,6 +599,10 @@ default Sender geoHashColumn(CharSequence name, long bits, int precisionBits) { *

      * The first call locks the column at {@code value.length() * 5} bits; all * subsequent rows must supply strings of the same length. + * In schema-aware mode, the server column must be a GEOHASH with exactly that + * precision, STRING, or VARCHAR. Text targets receive the corresponding binary + * bit string. Legacy mode keeps the native GEOHASH wire representation and + * server-side conversion behavior. * * @param name name of the column * @param value base32 geohash string, 1..12 characters; must not be null or empty diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8aeb01e2c..7c21812b0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2029,7 +2029,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "GEOHASH"); + binding.geoHashColumn(columnName, bits, precisionBits); return this; } if (precisionBits < 1 || precisionBits > 60) { @@ -2040,6 +2040,9 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int if (col != null) { col.addGeoHash(maskGeoHashBits(bits, precisionBits), precisionBits); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -2068,9 +2071,12 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(columnName, "GEOHASH"); + binding.geoHashColumn(columnName, value); return this; } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { rollbackRow(); throw e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 763095a1a..74c331bf8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -333,6 +333,58 @@ public QwpSchemaBinding floatColumn(CharSequence name, float value) { return floatingNumericColumn(name, value, "FLOAT"); } + public QwpSchemaBinding geoHashColumn(CharSequence name, long bits, int precisionBits) { + buffer.requireSchemaBinding(this); + if (precisionBits < 1 || precisionBits > 60) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); + } + int sourceType = ColumnType.getGeoHashTypeWithBits(precisionBits); + int index = targetIndex(name, "GEOHASH"); + int targetType = targetType(index, sourceType); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "GEOHASH", index, targetType); + if (column == null) { + return this; + } + long value = bits & ((1L << precisionBits) - 1L); + if (ColumnType.isGeoHash(targetType)) { + int targetBits = ColumnType.getGeoHashBits(targetType); + if (targetBits != precisionBits) { + throw unsupported(name, "GEOHASH", targetType, + "GEOHASH precision mismatch [sourceBits=" + precisionBits + ", targetBits=" + targetBits + ']'); + } + column.addGeoHash(value, precisionBits); + } else if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + column.addString(formatGeoHash(value, precisionBits)); + } else { + throw unsupported(name, "GEOHASH", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding geoHashColumn(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + if (value == null) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "GEOHASH string cannot be null; omit the setter to write NULL"); + } + int length = value.length(); + if (length == 0) { + throw error(INVALID_VALUE, name, "GEOHASH", null, "GEOHASH string cannot be empty"); + } + if (length > 12) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "GEOHASH string exceeds 12 characters: " + length); + } + final long bits; + try { + bits = Numbers.parseGeoHashBase32(value, 0, length); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "GEOHASH", null, "invalid GEOHASH string"); + } + return geoHashColumn(name, bits, length * 5); + } + public QwpSchemaBinding intColumn(CharSequence name, int value) { return integerNumericColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); } @@ -1159,6 +1211,19 @@ private CharSequence formatLong256(long l0, long l1, long l2, long l3) { return sink; } + private CharSequence formatGeoHash(long value, int precisionBits) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(60); + } else { + sink.clear(); + } + for (int bit = precisionBits - 1; bit >= 0; bit--) { + sink.putAscii(((value >>> bit) & 1) == 0 ? '0' : '1'); + } + return sink; + } + private CharSequence formatIPv4(int value) { StringSink sink = numericTextSink; if (sink == null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 5d6d89b97..857c8ffe7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -909,6 +909,66 @@ public void testIpv4GenerationRemainsNativeBeforeVarcharRebind() throws Exceptio }); } + @Test + public void testGeoHashTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + int[] targets = {geo20, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .geoHashColumn("value", "u33d") + .atNow(); + + sender.table("events").geoHashColumn("value", 0x11111, 20); + try { + sender.geoHashColumn("failed_b", 1, 20); + Assert.fail("expected GEOHASH to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.geoHashColumn("value", 0x12345, 20).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).geoHashTargetTable( + "events", 251 + target, 261, target, 0xd0c6c, 0x12345, 20); + Assert.assertEquals(2, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testGeoHashGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + LongTextSchemaHandler handler = new LongTextSchemaHandler(281, 291, geo20); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").geoHashColumn("value", 0xabcde, 20).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 292; + sender.table("events"); + try { + sender.stringColumn("value", "not-geohash"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.geoHashColumn("value", 0x12345, 20).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).twoNativeGeoHashAndVarcharBlocks( + "events", 281, 291, 292); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + @Test public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception { assertMemoryLeak(() -> { @@ -1293,6 +1353,23 @@ public void testOldPeerUsesLegacyLong256WireAndNeverDescribes() throws Exception }); } + @Test + public void testOldPeerUsesLegacyGeoHashWireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").geoHashColumn("value", 3, 5).atNow(); + sender.table("events").geoHashColumn("value", "U").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacyGeoHashTable("events", "value"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + @Test public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() throws Exception { assertMemoryLeak(() -> { @@ -1996,6 +2073,64 @@ private void geoHashTable( eof(); } + private void geoHashTargetTable( + String table, + int tableId, + long version, + int targetType, + long first, + long second, + int precision + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = ColumnType.isGeoHash(targetType) + ? QwpConstants.TYPE_GEOHASH + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, tableId, version, 2, "value", wireType); + if (wireType == QwpConstants.TYPE_GEOHASH) { + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(precision, varint()); + for (long value : new long[]{first, second}) { + for (int i = 0; i < (precision + 7) / 8; i++) { + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), u8()); + } + } + } else { + String firstText = "11010000110001101100"; + String secondText = "00010010001101000101"; + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(firstText.length(), in.getInt()); + Assert.assertEquals(firstText.length() + secondText.length(), in.getInt()); + Assert.assertEquals(firstText, stringBytes(firstText.length())); + Assert.assertEquals(secondText, stringBytes(secondText.length())); + } + eof(); + } + + private void legacyGeoHashTable(String table, String column) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, varint()); + Assert.assertEquals(3, u8()); + Assert.assertEquals(26, u8()); + eof(); + } + private void decimal64Table( String table, int tableId, long version, int scale, long first, long second ) { @@ -2240,6 +2375,28 @@ private void twoNativeLong256AndVarcharBlocks( eof(); } + private void twoNativeGeoHashAndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(20, varint()); + Assert.assertEquals(0xde, u8()); + Assert.assertEquals(0xbc, u8()); + Assert.assertEquals(0x0a, u8()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + String expected = "00010010001101000101"; + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + private void longTimestampTable( String table, int tableId, long version, int targetType, long expectedValue ) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index 81cb15baa..cf310f377 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -34,6 +34,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Numbers; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; @@ -1408,6 +1409,205 @@ public void testLong256DuplicatePrecedesUnsupportedAndRejectsParameterizedDesign } } + @Test + public void testNativeGeoHashConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_bits\tprecision_bits\texpected_value_hex\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + long input = Long.parseLong(fields[1]); + int precision = Integer.parseInt(fields[2]); + long expected = Long.parseUnsignedLong(fields[3], 16); + int geoType = ColumnType.getGeoHashTypeWithBits(precision); + for (int targetType : new int[]{geoType, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + rows.geoHashColumn("value", input, precision); + buffer.nextRow(); + byte wireType = ColumnType.isGeoHash(targetType) + ? QwpConstants.TYPE_GEOHASH + : QwpConstants.TYPE_VARCHAR; + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, wireType); + if (wireType == QwpConstants.TYPE_GEOHASH) { + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(precision, reader.varint()); + for (int i = 0; i < (precision + 7) / 8; i++) { + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.byteValue()); + } + } else { + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(fields[4].length(), reader.intValue()); + Assert.assertEquals(fields[4], reader.ascii(fields[4].length())); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(15, count); + }); + } + + @Test + public void testNativeGeoHashEveryPrecisionUsesFixedWidthText() throws Exception { + assertMemoryLeak(() -> { + for (int precision = 1; precision <= 60; precision++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + rows(buffer, column("value", ColumnType.STRING)) + .geoHashColumn("value", -1, precision); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(precision, reader.intValue()); + for (int i = 0; i < precision; i++) { + Assert.assertEquals('1', reader.byteValue()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testNativeGeoHashOmissionRollbackInferenceAndGuards() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", geo20), column("bad", ColumnType.UUID)); + rows.geoHashColumn("value", 0xabcde, 20) + .geoHashColumn("value", 0x12345, 20); + buffer.nextRow(); + buffer.nextRow(); + rows.geoHashColumn("value", 0x11111, 20); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("bad", 1, 20)); + rollbackCurrentRow(buffer); + rows.geoHashColumn("value", 0x12345, 20); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0x02, reader.byteValue()); + Assert.assertEquals(20, reader.varint()); + for (long value : new long[]{0xabcde, 0x12345}) { + for (int i = 0; i < 3; i++) { + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), reader.byteValue()); + } + } + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + rows.geoHashColumn("value", 0xabcde, 20); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(20, reader.varint()); + Assert.assertEquals(0xde, reader.byteValue()); + Assert.assertEquals(0xbc, reader.byteValue()); + Assert.assertEquals(0x0a, reader.byteValue()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("mismatch", ColumnType.getGeoHashTypeWithBits(15)), + column("unsupported", ColumnType.UUID), + column("parameterized", geo20, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("mismatch", 1, 0)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("mismatch", 1, 61)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("mismatch", 1, 20)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("unsupported", 1, 20)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("parameterized", 1, 20)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("ts", 1, 20)); + } + }); + } + + @Test + public void testNativeGeoHashTextOverloadPreservesSourcePrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("geo", ColumnType.getGeoHashTypeWithBits(20)), + column("text", ColumnType.STRING)); + rows.geoHashColumn("geo", "u33d") + .geoHashColumn("text", "U"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, + QwpConstants.TYPE_GEOHASH, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(20, reader.varint()); + long expected = Numbers.parseGeoHashBase32("u33d"); + for (int i = 0; i < 3; i++) { + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.byteValue()); + } + Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(5, reader.intValue()); + Assert.assertEquals("11010", reader.ascii(5)); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", ColumnType.getGeoHashTypeWithBits(15))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("value", "u33d")); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", (CharSequence) null)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "")); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "0123456789bcd")); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "a")); + } + }); + } + private static void assertCtorReason(int result, LineSenderSchemaException.Reason reason, String message) { try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv new file mode 100644 index 000000000..484515cc0 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv @@ -0,0 +1,16 @@ +# case_id input_bits precision_bits expected_value_hex expected_text +zero_1 0 1 0 0 +one_1 1 1 1 1 +zero_5 0 5 0 00000 +masked_5 -1 5 1f 11111 +max_7 127 7 7f 1111111 +max_8 255 8 ff 11111111 +max_15 32767 15 7fff 111111111111111 +max_16 65535 16 ffff 1111111111111111 +leading_zero_20 74565 20 12345 00010010001101000101 +pattern_20 703710 20 abcde 10101011110011011110 +max_31 2147483647 31 7fffffff 1111111111111111111111111111111 +max_32 4294967295 32 ffffffff 11111111111111111111111111111111 +max_35 34359738367 35 7ffffffff 11111111111111111111111111111111111 +masked_20 1152921504606846975 20 fffff 11111111111111111111 +max_60 1152921504606846975 60 fffffffffffffff 111111111111111111111111111111111111111111111111111111111111 diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index f90b1d106..92bb38e0b 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7326,3 +7326,68 @@ missed the separate `l1` formatting branch. Re-review found both resolved and no remaining correctness, compatibility, allocation or complexity issue in this slice. The wider conversion inventory remains open; the next bounded slice is native GEOHASH identity and text output with exact target precision. + +## D060 — GEOHASH uses exact native precision or fixed-width text + +Status: iteration 2.39c implemented and locally validated, 2026-09-14. Design +revision 62. The pre-iteration baseline is client `0d9663b26f` and server +`de96ef5eb5`; neither repository is committed by this decision record. + +Both public `geoHashColumn` overloads now participate in schema mode. A GEOHASH +target receives the masked packed value only when the target and source bit +precisions match exactly; a mismatch fails locally with `UNSUPPORTED_FEATURE`. +STRING and VARCHAR targets receive direct VARCHAR wire data containing exactly +one `0` or `1` per source bit. This matches the server cursor's reachable +positive-precision formatter rather than its unreachable character-formatting +branch. The implementation keeps one primitive long and its precision and +reuses the existing text sink. It adds no geohash object, lookup table, +converter registry, retained source value, server production change or per-row +string allocation. + +The packed overload accepts precision 1..60 and ignores higher input bits. The +textual overload preserves the existing public setter contract: strict, +case-insensitive base32 of 1..12 characters and five bits per character; null, +empty, overlong and invalid values fail locally. Omission is the only NULL +operation. All packed bit patterns are therefore values, including `0xff` at +eight bits. Schema-directed GEOHASH columns use the bitmap, so those +byte-aligned all-one values cannot collide with the fixed-width cursor sentinel. +Confirmed missing columns still infer the source precision. Legacy mode keeps +the original unflagged native GEOHASH frame and performs no DESCRIBE. + +The 15-row shared corpus covers precisions 1, 5, 7, 8, 15, 16, 20, 31, 32, 35 +and 60, leading zeroes, all-one values and masking of higher bits. Client and +server copies are byte-identical with SHA-256 +`8468be87f7061c58862b7bbe47290f508485c5cbc61a4373b2c3932e59a225e6`. +Component tests assert all 60 text widths, exact target wire bytes and bitmap +semantics, textual parsing, inference, duplicate suppression, rollback and +typed rejection of precision mismatch, unsupported, parameterized and +designated targets. Public socket tests cover exact schema frames, an existing +buffer pinned through a GEOHASH-to-VARCHAR rebind, and old-peer framing. Four +real-server tests cover missing-table inference, all shared text vectors, +exact native values including eight-bit `0xff`, A/error/C recovery, persisted +frame inspection and byte-preserving replay. + +The old `testCoercionToGeoHashErrors` expected every unsupported value to reach +the server. That no longer tests the product contract once schema mode rejects +the setter. It now asserts typed local errors and a subsequent successful row; +server-rejection tests remain separate instead of weakening a shared helper to +accept either path. This is a test-contract update, not a server behavior +change. + +On JDK 25, an expanded client regression passes 294 tests with zero failures, +errors or skips. It covers schema binding and Sender integration, table-buffer +and encoder behavior, legacy Sender framing, constants, earlier STRING-to- +GEOHASH, LONG256 and UUID conversions, recovered-frame analysis and replay. +The current client artifact installs successfully. The focused server gate +passes 15 tests: the four new GEOHASH tests, three STRING-to-GEOHASH tests, six +public Sender GEOHASH tests and two existing GEOHASH-to-text server tests. All +test JVM scratch roots are under `/mnt/pcie5`. + +The full `QwpSenderE2ETest` remains a diagnostic rather than an acceptance +gate: 137 tests produce two failures and 38 errors, improved from three failures +and 41 errors at D059. GEOHASH is no longer among them. The remaining product +gaps begin with integer temporal/text/decimal conversions, INT-to-IPv4 and +DOUBLE arrays; several other errors are legacy tests that still expect a +delayed server rejection after the new client correctly fails at the setter. +The next slice should implement integer temporal target representation, not +mechanically rewrite all remaining error tests. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 054387cdd..a27b7a921 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 61. Scope: QWP v1 over +Status: implemented on the development branch, revision 62. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.39a. Iteration -2.39b (LONG256 identity and text targets) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.39b. Iteration +2.39c (GEOHASH identity and text targets) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,9 +39,10 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.39b adds `long256Column` for LONG256, STRING and -VARCHAR targets. Its pre-iteration baseline is client `21168def` and server -`d7b94647ae`; the LONG256 slice is the current locally validated increment. +Current checkpoint: iteration 2.39c adds both `geoHashColumn` overloads for an +exact-precision GEOHASH, STRING and VARCHAR targets. Its pre-iteration baseline +is client `0d9663b26f` and server `de96ef5eb5`; the GEOHASH slice is the current +locally validated increment. The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: @@ -133,6 +134,17 @@ conversion behavior. Shared client/server vectors, exact wire checks, public Sender ingestion, rollback, schema rebind, legacy framing and persisted-frame replay cover the slice. D059 records the decision and evidence. +Both `geoHashColumn` overloads now use the server schema in schema mode. A +GEOHASH target keeps the masked packed value only when its precision exactly +matches the source. STRING and VARCHAR receive the server's fixed-width binary +text, one character per source bit, through the existing reusable sink. Bits +above the declared precision remain ignored. Omission is the only GEOHASH null +operation; all packed values are valid, including byte-aligned all-one values. +Missing columns still infer the source precision, while legacy mode keeps its +original native frame. Shared vectors, exact wire checks, public-Sender +ingestion, local-error rollback, schema rebind, old-peer framing and persisted +frame replay cover the slice. D060 records the decision and evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -141,8 +153,8 @@ Eight real-server tests cover these operations, relevant/unrelated DDL, schema lookups, auto-flush, reset and close. Fifteen public-Sender socket tests separately cover exact wire encoding, legacy fallback, one-way upgrade, startup timeout/interruption, pinned generations and split frames. -The broader server regression selection still has 38 classified failures -after the 2.22 expectation/fixture repairs; +At the 2.22 checkpoint, the broader server regression selection still had 38 +classified failures; this is not a release-ready client. Compatibility with a non-confirming socket peer is tested; an actual older QuestDB distribution was not run in this slice. The fixed 30-second schema deadline is settled policy and is not part of the @@ -351,8 +363,9 @@ emit the bitmap for schema-directed nullable GEOHASH columns, even with no null rows: otherwise valid byte-aligned maximum values are mistaken for nulls. Legacy and low-level non-nullable encoding remain unchanged. Only canonical packed GEOHASH types are recognized; unknown flags retain typed rejection. -Native GEOHASH setters remain unsupported and missing strings still infer -VARCHAR. Validation passes 464 client tests plus two packaging checks, 67 +At that checkpoint native GEOHASH setters remained unsupported and missing +strings still inferred VARCHAR; iteration 2.39c completes the native setter +slice. Validation passes 464 client tests plus two packaging checks, 67 focused server tests, 331 regressions and three 94-test seeded repeats. All 7,210 public differential cases match. The original 333-test diagnostic has 22 errors, removing only STRING-to-GEOHASH. Scoped native STRING-to-VARCHAR @@ -955,7 +968,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | | `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | | `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | All decimals, text | None; this is not the `stringColumn` parser path | -| `geoHashColumn(long, bits)` and `geoHashColumn(CharSequence)` (GEOHASH with source bits) | None | Geohash with exactly matching bit precision, text | +| `geoHashColumn(long, bits)` and `geoHashColumn(CharSequence)` (GEOHASH with source bits) | Geohash with exactly matching bit precision, text | None | | `doubleArray` (all Java ranks and `DoubleArray`; DOUBLE_ARRAY) | None | DOUBLE array of the target rank; identity/shape validation, not element casting | | `longArray` (all Java ranks and `LongArray`; LONG_ARRAY) | None | No supported conversion contract established; see the existing-server gap below | @@ -1058,7 +1071,14 @@ new casts or copy the unresolved behaviours below. width and 16 hex digits for each lower limb. Legacy native wire behavior is unchanged, including its bitmap-dependent server-side text conversion. - **Geohashes:** wire precision must equal target bits; no native GEOHASH narrowing is - accepted. `stringColumn` uses `GeoHashes.fromStringTruncatingNl`: empty means + accepted. The packed overload masks bits above its declared 1..60-bit + precision. The textual GEOHASH overload retains its public contract: strict + base32, 1..12 characters, with five bits per character; null and empty values + are invalid, and omission writes NULL. Every packed bit pattern is a value. + Schema-mode GEOHASH buffers therefore carry the null bitmap so byte-aligned + all-one values cannot collide with the legacy cursor sentinel. Native + GEOHASH-to-text emits exactly one `0` or `1` per source bit. `stringColumn` + uses `GeoHashes.fromStringTruncatingNl`: empty means null, parse at most the first 12 base32 characters, require enough bits and truncate to target precision. Invalid consumed characters fail; characters beyond the 12-character limit are ignored. The public geohash text setter @@ -1642,6 +1662,11 @@ They do not change the compatibility contract. Normalize the four-limb sentinel to the target bitmap before append. Do not add `BigInteger`, a LONG256 object, a formatter registry, retained source values or per-row string allocation. +21. **Keep GEOHASH as a masked long plus precision.** Reuse native GEOHASH + storage for an exact precision match and the existing text sink for the + fixed-width bit string. The schema buffer's bitmap already separates NULL + from every packed value. Do not add a geohash object, lookup table, + converter registry, retained source value or per-row string allocation. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1657,7 +1682,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.39b +### Plan after iteration 2.39c The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1685,20 +1710,17 @@ would add timing and orchestration without improving the contract assertion. The gate adds no protocol mode, adapter, send-time transformation or production code. -The next implementation slice is: - -1. **Iteration group 2.39 — fixed-width identity and text targets.** IPv4 to - IPv4/STRING/VARCHAR is implemented in 2.39a, and LONG256 to - LONG256/STRING/VARCHAR is implemented in 2.39b. Next, land GEOHASH to the - exact same-precision GEOHASH/STRING/VARCHAR as a separate reviewable slice. - It gets a public-Sender real-server E2E, exact target wire checks, null/omission - decisions, rollback, refresh and SF replay. Preserve geohash bit precision - and the existing public overload semantics; do not use this group to approve - unrelated parser paths. - -After these slices, reassess the remaining inventory in this order: integer to -DATE/timestamps; integer to text/SYMBOL/decimals and INT-to-IPv4; -then DOUBLE arrays after rank metadata is proven end to end. +Iteration group 2.39 is complete: IPv4, LONG256 and GEOHASH each use +target-native identity or text representation, with separate public E2E, +exact-wire, rollback, rebind, old-peer and replay coverage. The group added no +new conversion framework or server production code. + +The next bounded slice is integer temporal targets: BYTE, SHORT and INT to DATE, +TIMESTAMP and TIMESTAMP_NS, plus LONG to DATE. Preserve each setter's source +null semantics and the server's raw-count contract; these are representation +selections, not unit conversions. Then implement integer to +text/SYMBOL/decimals and INT-to-IPv4, followed by DOUBLE arrays after rank +metadata is proven end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From 853c3292c1d57c25e8f4c65fc0d607510d74be79 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 17:24:27 +0200 Subject: [PATCH 09/51] feat(qwp): support integer temporal targets --- .../qwp/protocol/QwpSchemaBinding.java | 12 +- .../QwpSchemaSenderIntegrationTest.java | 105 ++++++++++++++++++ .../qwp/protocol/QwpSchemaBindingTest.java | 99 ++++++++++++++++- .../qwp/integer-temporal-conversions.tsv | 25 +++++ design/schema-aware-sender-journal.md | 74 ++++++++++++ design/schema-aware-sender.md | 74 +++++++----- 6 files changed, 361 insertions(+), 28 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 74c331bf8..279118026 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -459,6 +459,7 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { if (!isNumericTarget(targetType) && !isTextTarget(targetType) && !ColumnType.isDecimal(targetType) + && targetType != ColumnType.DATE && targetType != ColumnType.TIMESTAMP_MICRO && targetType != ColumnType.TIMESTAMP_NANO) { throw unsupported(name, "LONG", targetType, "conversion is not implemented"); @@ -503,6 +504,7 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { column.addInt((int) value); break; case ColumnType.LONG: + case ColumnType.DATE: case ColumnType.TIMESTAMP_MICRO: case ColumnType.TIMESTAMP_NANO: column.addLong(value); @@ -981,6 +983,8 @@ private byte wireType(int targetType) { return QwpConstants.TYPE_INT; case ColumnType.LONG: return QwpConstants.TYPE_LONG; + case ColumnType.DATE: + return QwpConstants.TYPE_DATE; case ColumnType.FLOAT: return QwpConstants.TYPE_FLOAT; case ColumnType.DOUBLE: @@ -1136,7 +1140,10 @@ private QwpSchemaBinding integerNumericColumn( if (column == null) { return this; } - if (!isNumericTarget(targetType)) { + if (!isNumericTarget(targetType) + && targetType != ColumnType.DATE + && targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO) { throw unsupported(name, inputType, targetType, "conversion is not implemented"); } if (sourceNull) { @@ -1160,6 +1167,9 @@ private QwpSchemaBinding integerNumericColumn( column.addInt((int) value); break; case ColumnType.LONG: + case ColumnType.DATE: + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: column.addLong(value); break; case ColumnType.FLOAT: diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 857c8ffe7..a25d08ffd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -146,6 +146,65 @@ public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() thr }); } + @Test + public void testIntegerTemporalGenerationRemainsDateBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(981, 991, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 11).atNow(); + + handler.targetType = ColumnType.TIMESTAMP_NANO; + handler.version = 992; + sender.table("events"); + try { + sender.stringColumn("value", "1970-01-01T00:00:00.000001Z"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.intColumn("value", 12).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()) + .twoDateAndTimestampBlocks("events", 981, 991, 992); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.DATE, ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(961 + i, 971 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -1).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) 2).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).integerTemporalTable( + "events", 961 + i, 971 + i, temporalWireType(target)); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + } + }); + } + @Test public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { assertMemoryLeak(() -> { @@ -1750,6 +1809,19 @@ private static byte numericWireType(int targetType) { } } + private static byte temporalWireType(int targetType) { + switch (targetType) { + case ColumnType.DATE: + return QwpConstants.TYPE_DATE; + case ColumnType.TIMESTAMP_MICRO: + return QwpConstants.TYPE_TIMESTAMP; + case ColumnType.TIMESTAMP_NANO: + return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: + throw new AssertionError(ColumnType.nameOf(targetType)); + } + } + private static Sender sender(TestWebSocketServer server) { return Sender.fromConfig("ws::addr=localhost:" + server.getPort() + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" @@ -2293,6 +2365,23 @@ private void smallIntegerNumericTable(String table, int tableId, long version, b eof(); } + private void integerTemporalTable(String table, int tableId, long version, byte wireType) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 5, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + if (wireType != QwpConstants.TYPE_DATE) { + Assert.assertEquals(0, u8()); + } + Assert.assertEquals(-1, i64()); + Assert.assertEquals(2, i64()); + Assert.assertEquals(3, i64()); + Assert.assertEquals(Long.MAX_VALUE, i64()); + eof(); + } + private void assertSmallIntegerValue(byte wireType, long value) { switch (wireType) { case QwpConstants.TYPE_BYTE: @@ -2335,6 +2424,22 @@ private void twoGeoHashBlocks(String table, int tableId, long firstVersion, long eof(); } + private void twoDateAndTimestampBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(12, i64()); + eof(); + } + private void twoLong256AndVarcharBlocks( String table, int tableId, long firstVersion, long secondVersion ) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index cf310f377..6491ff054 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -568,6 +568,57 @@ public void testSmallIntegerNumericConformanceCorpus() throws Exception { Assert.assertEquals(64, count); } + @Test + public void testIntegerTemporalConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + try { + long input = Long.parseLong(fields[2]); + int targetType = temporalColumnType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + appendIntegerTemporal(rows, fields[1], input, "value"); + buffer.nextRow(); + Reader reader = tableReader( + encoder, + encoder.encodeSchema(buffer), + 1, + temporalWireType(fields[3]) + ); + if ("".equals(fields[4])) { + Assert.assertEquals("NULL bitmap must be present", 1, reader.byteValue()); + Assert.assertEquals("row zero must be NULL", 1, reader.byteValue()); + } else { + Assert.assertEquals("value must be bitmap-present", 0, reader.byteValue()); + } + if (!"DATE".equals(fields[3])) { + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.byteValue()); + } + if (!"".equals(fields[4])) { + Assert.assertEquals(Long.parseLong(fields[4]), reader.longValue()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(24, count); + } + @Test public void testSmallIntegerDuplicateFirstAndFailureRollback() throws Exception { assertMemoryLeak(() -> { @@ -658,13 +709,17 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { QwpSchemaBinding rows = rows(buffer, column("uuid", ColumnType.UUID), - column("future", ColumnType.INT, new byte[]{1})); + column("future", ColumnType.INT, new byte[]{1}), + column("future_date", ColumnType.DATE, new byte[]{1})); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.byteColumn("uuid", (byte) 1)); rollbackCurrentRow(buffer); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.shortColumn("future", (short) 2)); rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("future_date", (byte) 3)); + rollbackCurrentRow(buffer); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.intColumn("uuid", Integer.MIN_VALUE)); } @@ -1656,6 +1711,30 @@ private static void appendSmallInteger( } } + private static void appendIntegerTemporal( + QwpSchemaBinding rows, + String inputType, + long value, + String column + ) { + switch (inputType) { + case "BYTE": + rows.byteColumn(column, (byte) value); + break; + case "SHORT": + rows.shortColumn(column, (short) value); + break; + case "INT": + rows.intColumn(column, (int) value); + break; + case "LONG": + rows.longColumn(column, value); + break; + default: + throw new AssertionError(inputType); + } + } + private static void assertNumericValue(Reader reader, String target, String expected) { switch (target) { case "BYTE": @@ -1711,6 +1790,15 @@ private static int timestampColumnType(String target) { } } + private static int temporalColumnType(String target) { + switch (target) { + case "DATE": return ColumnType.DATE; + case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; + case "TIMESTAMP_NS": return ColumnType.TIMESTAMP_NANO; + default: throw new AssertionError(target); + } + } + private static byte timestampWireType(String target) { switch (target) { case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; @@ -1719,6 +1807,15 @@ private static byte timestampWireType(String target) { } } + private static byte temporalWireType(String target) { + switch (target) { + case "DATE": return QwpConstants.TYPE_DATE; + case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; + case "TIMESTAMP_NS": return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: throw new AssertionError(target); + } + } + private static byte numericWireType(String target) { switch (target) { case "BYTE": return QwpConstants.TYPE_BYTE; diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv new file mode 100644 index 000000000..e203958cc --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv @@ -0,0 +1,25 @@ +# case_id input_type input_value target_type expected_value +byte_date_min BYTE -128 DATE -128 +byte_date_max BYTE 127 DATE 127 +byte_timestamp_min BYTE -128 TIMESTAMP -128 +byte_timestamp_max BYTE 127 TIMESTAMP 127 +byte_timestamp_ns_min BYTE -128 TIMESTAMP_NS -128 +byte_timestamp_ns_max BYTE 127 TIMESTAMP_NS 127 +short_date_min SHORT -32768 DATE -32768 +short_date_max SHORT 32767 DATE 32767 +short_timestamp_min SHORT -32768 TIMESTAMP -32768 +short_timestamp_max SHORT 32767 TIMESTAMP 32767 +short_timestamp_ns_min SHORT -32768 TIMESTAMP_NS -32768 +short_timestamp_ns_max SHORT 32767 TIMESTAMP_NS 32767 +int_date_null INT -2147483648 DATE +int_date_negative INT -1 DATE -1 +int_date_max INT 2147483647 DATE 2147483647 +int_timestamp_null INT -2147483648 TIMESTAMP +int_timestamp_negative INT -1 TIMESTAMP -1 +int_timestamp_max INT 2147483647 TIMESTAMP 2147483647 +int_timestamp_ns_null INT -2147483648 TIMESTAMP_NS +int_timestamp_ns_negative INT -1 TIMESTAMP_NS -1 +int_timestamp_ns_max INT 2147483647 TIMESTAMP_NS 2147483647 +long_date_null LONG -9223372036854775808 DATE +long_date_min_value LONG -9223372036854775807 DATE -9223372036854775807 +long_date_max LONG 9223372036854775807 DATE 9223372036854775807 diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 92bb38e0b..880a9d7ce 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7391,3 +7391,77 @@ DOUBLE arrays; several other errors are legacy tests that still expect a delayed server rejection after the new client correctly fails at the setter. The next slice should implement integer temporal target representation, not mechanically rewrite all remaining error tests. + +## D061 — Integer temporal targets keep raw target-unit counts + +Status: iteration 2.40 implemented and locally validated, 2026-09-14. Design +revision 63. The pre-iteration baseline is client `8932076f24` and server +`67e2c94f7a`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +BYTE, SHORT and INT setters now accept DATE, TIMESTAMP and TIMESTAMP_NS targets; +LONG also accepts DATE. This is target representation selection, not a unit +conversion. The client writes the integer unchanged as epoch milliseconds for +DATE, epoch microseconds for TIMESTAMP, or epoch nanoseconds for TIMESTAMP_NS. +`Integer.MIN_VALUE` and `Long.MIN_VALUE` retain their schema-mode source-null +meaning and become explicit bitmap NULLs. BYTE and SHORT have no source-null +sentinel. + +The implementation was cross-checked against `QwpWalAppender` and +`WalColumnarRowAppender`. The server admits BYTE, SHORT, INT and LONG into DATE, +and routes integer input for DATE and both timestamp types through the same raw +64-bit append. It performs no scale multiplication or division. The client +therefore adds DATE to the existing target-wire switch, adds the three temporal +targets to the existing small-integer helper, and adds DATE to the established +LONG path. No converter registry, temporal object, unit layer, retained value, +per-row allocation or server production change was added. Numeric identity +targets still short-circuit at the existing `isNumericTarget` check, so the new +temporal comparisons are not evaluated on that hot path; this is source-level +reasoning, not a new performance claim. + +The first component test draft incorrectly expected the timestamp raw-encoding +discriminator after a DATE column. Inspecting `QwpColumnWriter` showed that +DATE is a plain fixed-width value, while TIMESTAMP and TIMESTAMP_NS carry the +discriminator. The test was corrected; production code was unchanged. This +wire difference is why the implementation deliberately does not introduce a +shared temporal encoder. + +Both repositories consume the same 24-case corpus, SHA-256 +`0872eab8b83f608c2e12778e4e44512c87bb7cff7e693109aed7effacd42fd85`. +It covers BYTE and SHORT extrema for all three targets, INT null/negative/max +for all three targets, and LONG null/minimum-non-null/max for DATE. Client +component tests assert exact target wire types, bitmap state, timestamp +discriminators and raw values. Public scripted-Sender tests cover all four +setters, a failed partial row followed by a valid row, the one-refresh path, +parameterized DATE rejection, and a DATE block remaining pinned when the next +schema generation changes the target to TIMESTAMP_NS. The real-server test +writes the shared corpus through public Sender setters, waits for ACK, and +checks the stored raw values and nulls through SQL. + +Focused client validation passes 106 tests, and the expanded regression passes +297 tests with zero failures, errors or skips. Packaged-JAR verification passes +46 selected unit tests and both packaging integration tests. The focused server +gate passes the new corpus plus the existing TIMESTAMP and TIMESTAMP_NS +aggregate tests: three tests total, all green. All test JVM scratch roots are +under `/mnt/pcie5`. + +The released-binary matrix also passes all three combinations with the freshly +packaged client: current client to QuestDB 10.0.1 remains legacy, client 1.3.9 +to the current server remains legacy, and current client to current server uses +schema mode. The run used +`/mnt/pcie5/qwp-integer-temporal-compat/run.FKXeM7`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 37 errors, down from two failures and 38 errors at D060. The +existing TIMESTAMP aggregate is now green. The DATE aggregate still stops at +the separate STRING-to-DATE gap after its integer setters succeed, so it is not +claimed as this slice's acceptance test. Other remaining errors include integer +text/decimal targets, INT-to-IPv4, arrays and legacy tests whose expected +server-side rejection is now a local setter error. + +No setter-specific store-and-forward scenario was added. This slice changes +only construction of ordinary schema-pinned bytes; persisted framing and replay +remain opaque to column type. Exact public wire tests prove the new DATE and +timestamp bytes before publication, while the existing schema-frame replay +suite remains green. Adding a second recovery harness would duplicate those +contracts without exercising changed recovery code. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index a27b7a921..cc686dcc1 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 62. Scope: QWP v1 over +Status: implemented on the development branch, revision 63. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.39b. Iteration -2.39c (GEOHASH identity and text targets) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.39c. Iteration +2.40 (integer temporal targets) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,10 +39,11 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.39c adds both `geoHashColumn` overloads for an -exact-precision GEOHASH, STRING and VARCHAR targets. Its pre-iteration baseline -is client `0d9663b26f` and server `de96ef5eb5`; the GEOHASH slice is the current -locally validated increment. +Current checkpoint: iteration 2.40 adds BYTE, SHORT and INT input to DATE, +TIMESTAMP and TIMESTAMP_NS targets, plus LONG input to DATE. Its pre-iteration +baseline is client `8932076f24` and server `67e2c94f7a`, whose submodule pins +that exact client revision. The integer temporal slice is the current locally +validated increment. The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: @@ -145,6 +146,17 @@ original native frame. Shared vectors, exact wire checks, public-Sender ingestion, local-error rollback, schema rebind, old-peer framing and persisted frame replay cover the slice. D060 records the decision and evidence. +BYTE, SHORT and INT inputs now select DATE, TIMESTAMP and TIMESTAMP_NS target +wire types directly; LONG also selects DATE. Values remain raw counts in the +target's unit. There is no implicit unit conversion: DATE means epoch +milliseconds, TIMESTAMP means epoch microseconds and TIMESTAMP_NS means epoch +nanoseconds. INT and LONG source sentinels become explicit bitmap NULLs; BYTE +and SHORT have no source-null sentinel. One existing small-integer helper and +the established LONG switch perform the append, with no new converter, retained +state, allocation or server production code. Shared exact-wire and real-server +vectors, partial-row rollback, parameter rejection and a DATE-to-TIMESTAMP_NS +schema rebind cover the slice. D061 records the decision and evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -921,9 +933,9 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.38 on 2026-09-14. The accepted baseline is client -`1310b0dd` and server `8ad0b92baf`, whose submodule pins that exact client -revision. The original audit bases were client +refreshed through iteration 2.40 on 2026-09-14. The pre-iteration baseline is +client `8932076f24` and server `67e2c94f7a`, whose submodule pins that exact +client revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; runtime evidence for implemented slices is recorded separately in the journal. @@ -953,9 +965,9 @@ non-null conversion. Known compatibility exceptions above still apply. | Public input (source wire type) | Implemented targets | Missing server-accepted targets | | --- | --- | --- | | `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | -| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric | DATE, timestamps, text, SYMBOL, decimals | -| `intColumn` (INT) | Numeric | DATE, timestamps, text, SYMBOL, decimals, IPv4 | -| `longColumn` (LONG) | Numeric, text, SYMBOL, timestamps, decimals | DATE | +| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps | Text, SYMBOL, decimals | +| `intColumn` (INT) | Numeric, DATE, timestamps | Text, SYMBOL, decimals, IPv4 | +| `longColumn` (LONG) | Numeric, text, SYMBOL, DATE, timestamps, decimals | None | | `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | | `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | | `symbol` (SYMBOL) | Text, SYMBOL | None | @@ -976,13 +988,12 @@ DATE also exists as a wire input, but there is no public WebSocket `dateColumn` setter. The server accepts DATE-to-DATE and DATE-to-text only. Neither has a binding entrypoint today; record this wire-only coverage without adding a new public API just to mirror the protocol. DATE **targets** from existing integer -and string setters are real missing work in the table. +setters are implemented in iteration 2.40; STRING-to-DATE remains missing. -The binding also lacks target representation selection for DATE and arrays. It -rejects extension parameters. -Adding a parser alone therefore does not complete these pairs: select the -target wire representation and pin its parameters before append. Preserve the -existing non-owning binding and buffer ownership. +The binding still lacks target representation selection for arrays and rejects +extension parameters. Adding array append code alone therefore does not +complete those pairs: select the target wire representation and pin its rank +before append. Preserve the existing non-owning binding and buffer ownership. ### Rules the missing conversions must cover @@ -1667,6 +1678,12 @@ They do not change the compatibility contract. fixed-width bit string. The schema buffer's bitmap already separates NULL from every packed value. Do not add a geohash object, lookup table, converter registry, retained source value or per-row string allocation. +22. **Treat integer temporal targets as representation selection.** DATE, + TIMESTAMP and TIMESTAMP_NS all retain the integer value as a raw 64-bit + count; only the target wire type and its unit differ. Reuse the existing + integer append paths and source-null rules. Do not add a temporal value + object, unit-normalization layer or shared encoder: DATE has no timestamp + encoding discriminator, while both timestamp wire types do. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1682,7 +1699,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.39c +### Plan after iteration 2.40 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1715,12 +1732,17 @@ target-native identity or text representation, with separate public E2E, exact-wire, rollback, rebind, old-peer and replay coverage. The group added no new conversion framework or server production code. -The next bounded slice is integer temporal targets: BYTE, SHORT and INT to DATE, -TIMESTAMP and TIMESTAMP_NS, plus LONG to DATE. Preserve each setter's source -null semantics and the server's raw-count contract; these are representation -selections, not unit conversions. Then implement integer to -text/SYMBOL/decimals and INT-to-IPv4, followed by DOUBLE arrays after rank -metadata is proven end to end. +Iteration 2.40 is complete locally: BYTE, SHORT and INT select DATE, TIMESTAMP +and TIMESTAMP_NS target representations, and LONG selects DATE. Source values +remain raw counts in the target unit; no unit conversion or new temporal +abstraction was added. + +The next bounded slice is BYTE, SHORT and INT to STRING, VARCHAR and SYMBOL. +Reuse the existing allocation-free integer formatter and text/symbol append +paths. Then reuse the established exact decimal append tail for those three +integer sources, implement the separate INT-to-IPv4 null translation, and add +STRING-to-DATE parsing. DOUBLE arrays follow only after rank metadata is proven +end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From 3acdceadbd3203c7c7b2ef5a5723cff0c6f304d5 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 18:05:12 +0200 Subject: [PATCH 10/51] feat(qwp): support small integer text targets --- .../qwp/protocol/QwpSchemaBinding.java | 18 ++- .../QwpSchemaSenderIntegrationTest.java | 139 +++++++++++++++++- .../qwp/protocol/QwpSchemaBindingTest.java | 86 ++++++++++- .../cutlass/qwp/small-integer-to-text.tsv | 31 ++++ design/schema-aware-sender-journal.md | 64 ++++++++ design/schema-aware-sender.md | 58 +++++--- 6 files changed, 368 insertions(+), 28 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 279118026..a2b55ef90 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -265,7 +265,7 @@ public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { } public QwpSchemaBinding byteColumn(CharSequence name, byte value) { - return integerNumericColumn(name, value, "BYTE", ColumnType.BYTE, false); + return integerColumn(name, value, "BYTE", ColumnType.BYTE, false); } public QwpSchemaBinding doubleColumn(CharSequence name, double value) { @@ -386,7 +386,7 @@ public QwpSchemaBinding geoHashColumn(CharSequence name, CharSequence value) { } public QwpSchemaBinding intColumn(CharSequence name, int value) { - return integerNumericColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); + return integerColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); } public QwpSchemaBinding ipv4Column(CharSequence name, int address) { @@ -553,7 +553,7 @@ public QwpSchemaBinding long256Column(CharSequence name, long l0, long l1, long } public QwpSchemaBinding shortColumn(CharSequence name, short value) { - return integerNumericColumn(name, value, "SHORT", ColumnType.SHORT, false); + return integerColumn(name, value, "SHORT", ColumnType.SHORT, false); } public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { @@ -1126,7 +1126,7 @@ private void parseDecimalText(CharSequence name, CharSequence value, Decimal256 } } - private QwpSchemaBinding integerNumericColumn( + private QwpSchemaBinding integerColumn( CharSequence name, long value, String inputType, @@ -1141,6 +1141,7 @@ private QwpSchemaBinding integerNumericColumn( return this; } if (!isNumericTarget(targetType) + && !isTextTarget(targetType) && targetType != ColumnType.DATE && targetType != ColumnType.TIMESTAMP_MICRO && targetType != ColumnType.TIMESTAMP_NANO) { @@ -1178,8 +1179,15 @@ private QwpSchemaBinding integerNumericColumn( case ColumnType.DOUBLE: column.addDouble((double) value); break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatLong(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatLong(value)); + break; default: - throw new AssertionError("unsupported numeric target"); + throw new AssertionError("unsupported integer target"); } return this; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index a25d08ffd..1969e4a01 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -205,6 +205,64 @@ public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exc }); } + @Test + public void testSmallIntegerTextGenerationRemainsSymbolBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1021, 1031, ColumnType.SYMBOL); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 7).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 1032; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 2); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.shortColumn("value", (short) 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()) + .twoSymbolAndVarcharBlocks("events", 1021, 1031, 1032); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(1041 + i, 1051 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -128).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", Short.MAX_VALUE).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerTextTable( + "events", 1041 + i, 1051 + i, target); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + } + }); + } + @Test public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { assertMemoryLeak(() -> { @@ -1140,7 +1198,7 @@ public void testLongSymbolGenerationRemainsPinnedAcrossTextRebind() throws Excep sender.flush(); new FrameReader(handler.awaitDataFrame()) - .twoLongTextBlocks("events", 151, 161, 162); + .twoSymbolAndVarcharBlocks("events", 151, 161, 162); Assert.assertEquals(2, handler.describeRequests.get()); } }); @@ -1376,6 +1434,26 @@ public void testOldPeerUsesLegacyFrameAndNeverDescribes() throws Exception { }); } + @Test + public void testOldPeerUsesLegacySmallIntegerWireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .byteColumn("b", (byte) -7) + .shortColumn("s", (short) 42) + .intColumn("i", Integer.MIN_VALUE) + .atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacySmallIntegerTable("events"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + @Test public void testOldPeerUsesLegacyIpv4WireAndNeverDescribes() throws Exception { assertMemoryLeak(() -> { @@ -2027,6 +2105,32 @@ private void legacyLong256Table(String table, String column) { eof(); } + private void legacySmallIntegerTable(String table) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(3, varint()); + Assert.assertEquals("b", string()); + Assert.assertEquals(QwpConstants.TYPE_BYTE, u8()); + Assert.assertEquals("s", string()); + Assert.assertEquals(QwpConstants.TYPE_SHORT, u8()); + Assert.assertEquals("i", string()); + Assert.assertEquals(QwpConstants.TYPE_INT, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals((byte) -7 & 0xff, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(42, in.getShort()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Integer.MIN_VALUE, in.getInt()); + eof(); + } + private void longTextTable( String table, int tableId, @@ -2365,6 +2469,37 @@ private void smallIntegerNumericTable(String table, int tableId, long version, b eof(); } + private void smallIntegerTextTable(String table, int tableId, long version, int targetType) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(4, varint()); + Assert.assertEquals("-128", string()); + Assert.assertEquals("99", string()); + Assert.assertEquals("32767", string()); + Assert.assertEquals("2147483647", string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, tableId, version, 4, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(3, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(4, in.getInt()); + Assert.assertEquals(9, in.getInt()); + Assert.assertEquals(19, in.getInt()); + Assert.assertEquals("-128327672147483647", stringBytes(19)); + } + eof(); + } + private void integerTemporalTable(String table, int tableId, long version, byte wireType) { messageHeader(1); Assert.assertEquals(0, varint()); @@ -2682,7 +2817,7 @@ private void schemaBlockHeader( Assert.assertEquals(wireType & 0xff, u8()); } - private void twoLongTextBlocks(String table, int tableId, long firstVersion, long secondVersion) { + private void twoSymbolAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { messageHeader(2); Assert.assertEquals(0, varint()); Assert.assertEquals(1, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index 6491ff054..b3edcf4c4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -568,6 +568,65 @@ public void testSmallIntegerNumericConformanceCorpus() throws Exception { Assert.assertEquals(64, count); } + @Test + public void testSmallIntegerTextConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try { + long input = Long.parseLong(fields[2]); + byte wireType = textWireType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", textColumnType(fields[3]))); + appendSmallInteger(rows, fields[1], input, "value"); + buffer.nextRow(); + Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + if ("".equals(fields[4])) { + Assert.assertEquals("", fields[5]); + Assert.assertEquals("NULL bitmap must be present", 1, reader.byteValue()); + Assert.assertEquals("row zero must be NULL", 1, reader.byteValue()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.intValue()); + } + } else { + byte[] expected = hexBytes(fields[4]); + Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals("value must be bitmap-present", 0, reader.byteValue()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(fields[5], reader.stringValue()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(expected.length, reader.intValue()); + for (byte value : expected) { + Assert.assertEquals(value & 0xff, reader.byteValue()); + } + } + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(30, count); + } + @Test public void testIntegerTemporalConformanceCorpusUsesTargetWire() throws Exception { InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( @@ -710,7 +769,8 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() QwpSchemaBinding rows = rows(buffer, column("uuid", ColumnType.UUID), column("future", ColumnType.INT, new byte[]{1}), - column("future_date", ColumnType.DATE, new byte[]{1})); + column("future_date", ColumnType.DATE, new byte[]{1}), + column("future_text", ColumnType.VARCHAR, new byte[]{1})); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.byteColumn("uuid", (byte) 1)); rollbackCurrentRow(buffer); @@ -720,6 +780,9 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.byteColumn("future_date", (byte) 3)); rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("future_text", 4)); + rollbackCurrentRow(buffer); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.intColumn("uuid", Integer.MIN_VALUE)); } @@ -1735,6 +1798,14 @@ private static void appendIntegerTemporal( } } + private static byte[] hexBytes(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + private static void assertNumericValue(Reader reader, String target, String expected) { switch (target) { case "BYTE": @@ -1799,6 +1870,19 @@ private static int temporalColumnType(String target) { } } + private static int textColumnType(String target) { + switch (target) { + case "STRING": return ColumnType.STRING; + case "VARCHAR": return ColumnType.VARCHAR; + case "SYMBOL": return ColumnType.SYMBOL; + default: throw new AssertionError(target); + } + } + + private static byte textWireType(String target) { + return "SYMBOL".equals(target) ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR; + } + private static byte timestampWireType(String target) { switch (target) { case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv new file mode 100644 index 000000000..32b77a1b5 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv @@ -0,0 +1,31 @@ +# case_id input_type input_value target_type expected_utf8_hex expected_sql_value +byte_string_min BYTE -128 STRING 2d313238 -128 +byte_string_max BYTE 127 STRING 313237 127 +short_string_min SHORT -32768 STRING 2d3332373638 -32768 +short_string_max SHORT 32767 STRING 3332373637 32767 +int_string_null INT -2147483648 STRING +int_string_min_value INT -2147483647 STRING 2d32313437343833363437 -2147483647 +int_string_neg_one INT -1 STRING 2d31 -1 +int_string_zero INT 0 STRING 30 0 +int_string_one INT 1 STRING 31 1 +int_string_max INT 2147483647 STRING 32313437343833363437 2147483647 +byte_varchar_min BYTE -128 VARCHAR 2d313238 -128 +byte_varchar_max BYTE 127 VARCHAR 313237 127 +short_varchar_min SHORT -32768 VARCHAR 2d3332373638 -32768 +short_varchar_max SHORT 32767 VARCHAR 3332373637 32767 +int_varchar_null INT -2147483648 VARCHAR +int_varchar_min_value INT -2147483647 VARCHAR 2d32313437343833363437 -2147483647 +int_varchar_neg_one INT -1 VARCHAR 2d31 -1 +int_varchar_zero INT 0 VARCHAR 30 0 +int_varchar_one INT 1 VARCHAR 31 1 +int_varchar_max INT 2147483647 VARCHAR 32313437343833363437 2147483647 +byte_symbol_min BYTE -128 SYMBOL 2d313238 -128 +byte_symbol_max BYTE 127 SYMBOL 313237 127 +short_symbol_min SHORT -32768 SYMBOL 2d3332373638 -32768 +short_symbol_max SHORT 32767 SYMBOL 3332373637 32767 +int_symbol_null INT -2147483648 SYMBOL +int_symbol_min_value INT -2147483647 SYMBOL 2d32313437343833363437 -2147483647 +int_symbol_neg_one INT -1 SYMBOL 2d31 -1 +int_symbol_zero INT 0 SYMBOL 30 0 +int_symbol_one INT 1 SYMBOL 31 1 +int_symbol_max INT 2147483647 SYMBOL 32313437343833363437 2147483647 diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 880a9d7ce..65f2c03b1 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7465,3 +7465,67 @@ remain opaque to column type. Exact public wire tests prove the new DATE and timestamp bytes before publication, while the existing schema-frame replay suite remains green. Adding a second recovery harness would duplicate those contracts without exercising changed recovery code. + +## D062 — Small integers reuse the established text and symbol paths + +Status: iteration 2.41 implemented and locally validated, 2026-09-14. Design +revision 64. The pre-iteration baseline is client `853c3292c1` and server +`85f24785d3`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +BYTE, SHORT and INT setters now accept STRING, VARCHAR and SYMBOL targets in +schema mode. Each input widens losslessly to long and uses the existing reusable +numeric text sink. STRING and VARCHAR append direct VARCHAR wire data; SYMBOL +uses the existing global dictionary writer. `Integer.MIN_VALUE` retains its +source-null meaning and becomes an explicit target bitmap NULL. BYTE and SHORT +have no source-null sentinel. Missing columns still infer their native input +types, and a legacy peer still receives native BYTE, SHORT and INT wire columns +without a DESCRIBE. + +The implementation was cross-checked against the current server's +`QwpWalAppender` dispatch and `WalColumnarRowAppender`. Fixed-width integer +cursors targeting STRING, VARCHAR or SYMBOL all call `Numbers.append` on +`cursor.getLong()`. The client therefore extends the existing small-integer +target switch and reuses `formatLong`, `addString` and `addSymbol`. It adds no +source-width formatter, converter object, retained value, per-row `String`, +protocol field or server production change. Numeric identity targets still +short-circuit at the first `isNumericTarget` check, so the additional text check +is not evaluated on that hot path. This is source-level reasoning, not a new +performance claim. + +Client and server consume the same 30-case corpus, SHA-256 +`78481d27bbeb70354f309372650311e269ee365087e9003218222c2b0580dcfe`. +For each target it covers BYTE and SHORT extrema plus INT null, +minimum-non-null, negative one, zero, one and maximum. Component tests assert +the exact target wire type, UTF-8 payload or symbol dictionary entry, offsets, +IDs and bitmap null. Public socket tests additionally cover a rolled-back symbol +dictionary entry, A/error/C row recovery, the standard one-refresh path, a +SYMBOL block pinned before a VARCHAR schema generation, parameter rejection and +old-peer native framing. + +The real-server E2E uses the public `Sender` API and one flush across STRING, +VARCHAR and SYMBOL tables, waits for the acknowledged sequence, drains WAL and +asserts exact SQL values and nullness for all 30 vectors. A second E2E proves a +partially written row is discarded after an INT-to-UUID local error, completed +rows survive and the same symbol value is reused. The focused server gate also +runs the existing `testCoercionToSymbol`; all three tests pass. + +Expanded client validation passes 305 tests with zero failures, errors or +skips. Packaged-JAR verification passes 47 selected unit tests and both +packaging integration tests. The released-binary matrix passes current client +to QuestDB 10.0.1 in legacy mode, client 1.3.9 to current server in legacy mode, +and current client to current server in schema mode. All scratch roots are +under `/mnt/pcie5`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 36 errors, down from 37 errors at D061. The SYMBOL aggregate is now +green. The STRING and VARCHAR aggregates advance through the small-integer +inputs and stop at the separate CHAR-to-text gap; this slice does not claim +that unresolved contract. Other open implementation entries include +small-integer decimal targets, INT-to-IPv4, STRING-to-DATE and DOUBLE arrays. + +No new store-and-forward test was added. The changed code only chooses an +already supported target-native VARCHAR or SYMBOL representation before the +frame is persisted. Exact public wire tests prove those bytes, while existing +schema-frame replay and the established long-to-text persistence coverage remain +green. A setter-specific replay test would repeat unchanged persistence code. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index cc686dcc1..85c5e2adf 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 63. Scope: QWP v1 over +Status: implemented on the development branch, revision 64. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.39c. Iteration -2.40 (integer temporal targets) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.40. Iteration +2.41 (small-integer text targets) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,11 +39,10 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.40 adds BYTE, SHORT and INT input to DATE, -TIMESTAMP and TIMESTAMP_NS targets, plus LONG input to DATE. Its pre-iteration -baseline is client `8932076f24` and server `67e2c94f7a`, whose submodule pins -that exact client revision. The integer temporal slice is the current locally -validated increment. +Current checkpoint: iteration 2.41 adds BYTE, SHORT and INT input to STRING, +VARCHAR and SYMBOL targets. Its pre-iteration baseline is client `853c3292c1` +and server `85f24785d3`, whose submodule pins that exact client revision. The +small-integer text slice is the current locally validated increment. The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: @@ -157,6 +156,17 @@ state, allocation or server production code. Shared exact-wire and real-server vectors, partial-row rollback, parameter rejection and a DATE-to-TIMESTAMP_NS schema rebind cover the slice. D061 records the decision and evidence. +BYTE, SHORT and INT inputs now also select STRING, VARCHAR and SYMBOL target +wire types directly. The existing reusable integer text sink formats the widened +value once; STRING and VARCHAR use the same VARCHAR wire representation, while +SYMBOL uses the existing global dictionary path. `Integer.MIN_VALUE` remains a +source NULL and is written through the target bitmap. BYTE and SHORT have no +source-null sentinel. No source-width formatter, converter object, retained +value, per-row String or server production code was added. A shared 30-case +corpus, exact-wire tests, generation rebind, legacy isolation, public-Sender +real-server ingestion and A/error/C rollback cover the slice. D062 records the +decision and evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -933,8 +943,8 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.40 on 2026-09-14. The pre-iteration baseline is -client `8932076f24` and server `67e2c94f7a`, whose submodule pins that exact +refreshed through iteration 2.41 on 2026-09-14. The pre-iteration baseline is +client `853c3292c1` and server `85f24785d3`, whose submodule pins that exact client revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; @@ -965,8 +975,8 @@ non-null conversion. Known compatibility exceptions above still apply. | Public input (source wire type) | Implemented targets | Missing server-accepted targets | | --- | --- | --- | | `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | -| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps | Text, SYMBOL, decimals | -| `intColumn` (INT) | Numeric, DATE, timestamps | Text, SYMBOL, decimals, IPv4 | +| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps, text, SYMBOL | Decimals | +| `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL | Decimals, IPv4 | | `longColumn` (LONG) | Numeric, text, SYMBOL, DATE, timestamps, decimals | None | | `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | | `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | @@ -1684,6 +1694,10 @@ They do not change the compatibility contract. integer append paths and source-null rules. Do not add a temporal value object, unit-normalization layer or shared encoder: DATE has no timestamp encoding discriminator, while both timestamp wire types do. +23. **Reuse one integer formatter for integer text targets.** BYTE, SHORT and + INT widen losslessly to long, so use the established reusable long sink and + direct STRING/VARCHAR or SYMBOL append paths. Do not add source-width + formatters, boxed values, per-row strings or a text conversion registry. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1699,7 +1713,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.40 +### Plan after iteration 2.41 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1732,17 +1746,21 @@ target-native identity or text representation, with separate public E2E, exact-wire, rollback, rebind, old-peer and replay coverage. The group added no new conversion framework or server production code. -Iteration 2.40 is complete locally: BYTE, SHORT and INT select DATE, TIMESTAMP +Iteration 2.40 is committed: BYTE, SHORT and INT select DATE, TIMESTAMP and TIMESTAMP_NS target representations, and LONG selects DATE. Source values remain raw counts in the target unit; no unit conversion or new temporal abstraction was added. -The next bounded slice is BYTE, SHORT and INT to STRING, VARCHAR and SYMBOL. -Reuse the existing allocation-free integer formatter and text/symbol append -paths. Then reuse the established exact decimal append tail for those three -integer sources, implement the separate INT-to-IPv4 null translation, and add -STRING-to-DATE parsing. DOUBLE arrays follow only after rank metadata is proven -end to end. +Iteration 2.41 is complete locally: BYTE, SHORT and INT select STRING, VARCHAR +and SYMBOL target representations through the existing reusable integer +formatter and direct text/symbol append paths. The real-server conversion +diagnostic improves by exactly one error because the SYMBOL aggregate is now +green; STRING and VARCHAR continue to the independent CHAR-to-text gap. + +The next bounded slice is BYTE, SHORT and INT to decimal targets. Reuse the +established exact decimal append tail rather than add another conversion path. +Then implement the separate INT-to-IPv4 null translation and STRING-to-DATE +parsing. DOUBLE arrays follow only after rank metadata is proven end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From 1630c04775940dbdfcf172eda97d26fc6efb50d9 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 18:34:10 +0200 Subject: [PATCH 11/51] feat(qwp): support small integer decimal targets --- .../qwp/protocol/QwpSchemaBinding.java | 42 +++-- .../QwpSchemaSenderIntegrationTest.java | 88 ++++++++++ .../qwp/protocol/QwpSchemaBindingTest.java | 159 +++++++++++++++++- .../cutlass/qwp/small-integer-to-decimal.tsv | 31 ++++ design/schema-aware-sender-journal.md | 64 +++++++ design/schema-aware-sender.md | 57 +++++-- 6 files changed, 410 insertions(+), 31 deletions(-) create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index a2b55ef90..a21dccfbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -465,19 +465,7 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { throw unsupported(name, "LONG", targetType, "conversion is not implemented"); } if (ColumnType.isDecimal(targetType)) { - int targetPrecision = ColumnType.getDecimalPrecision(targetType); - int targetScale = ColumnType.getDecimalScale(targetType); - if (value == Long.MIN_VALUE) { - column.addSchemaDecimalNull(targetScale); - return this; - } - try { - if (!column.addSchemaLongDecimal(value, targetPrecision, targetScale)) { - throw error(INVALID_VALUE, name, "LONG", targetType, "decimal value exceeds target precision"); - } - } catch (NumericException e) { - throw error(INVALID_VALUE, name, "LONG", targetType, "decimal value cannot be rescaled exactly"); - } + appendIntegerDecimal(column, name, value, "LONG", targetType, value == Long.MIN_VALUE); return this; } if (value == Long.MIN_VALUE) { @@ -1142,11 +1130,16 @@ private QwpSchemaBinding integerColumn( } if (!isNumericTarget(targetType) && !isTextTarget(targetType) + && !ColumnType.isDecimal(targetType) && targetType != ColumnType.DATE && targetType != ColumnType.TIMESTAMP_MICRO && targetType != ColumnType.TIMESTAMP_NANO) { throw unsupported(name, inputType, targetType, "conversion is not implemented"); } + if (ColumnType.isDecimal(targetType)) { + appendIntegerDecimal(column, name, value, inputType, targetType, sourceNull); + return this; + } if (sourceNull) { column.addNull(); return this; @@ -1192,6 +1185,29 @@ private QwpSchemaBinding integerColumn( return this; } + private void appendIntegerDecimal( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + long value, + String inputType, + int targetType, + boolean sourceNull + ) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (sourceNull) { + column.addSchemaDecimalNull(targetScale); + return; + } + try { + if (!column.addSchemaLongDecimal(value, targetPrecision, targetScale)) { + throw error(INVALID_VALUE, name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, inputType, targetType, "decimal value cannot be rescaled exactly"); + } + } + private CharSequence formatLong(long value) { StringSink sink = numericTextSink; if (sink == null) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 1969e4a01..9796525d8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -263,6 +263,62 @@ public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Ex }); } + @Test + public void testSmallIntegerDecimalGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1061, 1071, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 1).atNow(); + + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 1072; + sender.table("events"); + try { + sender.shortColumn("value", (short) 1_000); + Assert.fail("expected width-and-scale-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.shortColumn("value", (short) 1_000).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).twoSmallIntegerDecimalBlocks( + "events", 1061, 1071, 1072); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1081, 1091, ColumnType.getDecimalType(18, 2)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 42).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) -7).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerDecimalTable( + "events", 1081, 1091); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + }); + } + @Test public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { assertMemoryLeak(() -> { @@ -2469,6 +2525,20 @@ private void smallIntegerNumericTable(String table, int tableId, long version, b eof(); } + private void smallIntegerDecimalTable(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(2, u8()); + Assert.assertEquals(4_200, i64()); + Assert.assertEquals(-700, i64()); + Assert.assertEquals(214_748_364_700L, i64()); + eof(); + } + private void smallIntegerTextTable(String table, int tableId, long version, int targetType) { messageHeader(1); if (targetType == ColumnType.SYMBOL) { @@ -2835,6 +2905,24 @@ private void twoSymbolAndVarcharBlocks(String table, int tableId, long firstVers eof(); } + private void twoSmallIntegerDecimalBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, u8()); + Assert.assertEquals(100_000_000, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + private void twoUuidAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { messageHeader(2); Assert.assertEquals(0, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index b3edcf4c4..c4467e4d2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -627,6 +627,85 @@ public void testSmallIntegerTextConformanceCorpusUsesTargetWire() throws Excepti Assert.assertEquals(30, count); } + @Test + public void testSmallIntegerDecimalConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv"); + Assert.assertNotNull(stream); + boolean[][] sourceTargets = new boolean[3][6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals( + "# case_id\tinput_type\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome" + + "\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql", + lines.readLine() + ); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 12, fields.length); + try { + long input = Long.parseLong(fields[2]); + int precision = Integer.parseInt(fields[4]); + int scale = Integer.parseInt(fields[5]); + Assert.assertEquals(decimalStorageType(precision), fields[3]); + sourceTargets[smallIntegerSourceIndex(fields[1])][decimalStorageIndex(fields[3])] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows( + buffer, + column("value", ColumnType.getDecimalType(precision, scale)) + ); + Runnable append = () -> appendSmallInteger(rows, fields[1], input, "value"); + if ("INVALID".equals(fields[6])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + rollbackCurrentRow(buffer); + } else { + append.run(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader( + encoder, + size, + 1, + decimalWireType(fields[3]) + ); + if ("NULL".equals(fields[6])) { + Assert.assertEquals("NULL", fields[11]); + Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.byteValue()); + } else { + Assert.assertEquals("VALUE", fields[6]); + Assert.assertEquals(0, reader.byteValue()); + } + Assert.assertEquals(scale, reader.byteValue()); + if ("VALUE".equals(fields[6])) { + int limbs = decimalWireLongCount(fields[3]); + for (int i = 0; i < limbs; i++) { + Assert.assertEquals(parseHexLong(fields[7 + i]), reader.longValue()); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[7 + i]); + } + } + Assert.assertEquals(size, reader.position()); + } + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(30, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target coverage " + source + '/' + target, + sourceTargets[source][target]); + } + } + } + @Test public void testIntegerTemporalConformanceCorpusUsesTargetWire() throws Exception { InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( @@ -770,7 +849,8 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() column("uuid", ColumnType.UUID), column("future", ColumnType.INT, new byte[]{1}), column("future_date", ColumnType.DATE, new byte[]{1}), - column("future_text", ColumnType.VARCHAR, new byte[]{1})); + column("future_text", ColumnType.VARCHAR, new byte[]{1}), + column("future_decimal", ColumnType.getDecimalType(18, 2), new byte[]{1})); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.byteColumn("uuid", (byte) 1)); rollbackCurrentRow(buffer); @@ -783,6 +863,9 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.intColumn("future_text", 4)); rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("future_decimal", (short) 5)); + rollbackCurrentRow(buffer); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.intColumn("uuid", Integer.MIN_VALUE)); } @@ -1774,6 +1857,63 @@ private static void appendSmallInteger( } } + private static int decimalStorageIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + Assert.assertEquals("DECIMAL256", target); + return 5; + } + } + + private static String decimalStorageType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + + private static int decimalWireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static byte decimalWireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + private static void appendIntegerTemporal( QwpSchemaBinding rows, String inputType, @@ -1853,6 +1993,23 @@ private static int numericColumnType(String target) { } } + private static long parseHexLong(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int smallIntegerSourceIndex(String source) { + switch (source) { + case "BYTE": + return 0; + case "SHORT": + return 1; + default: + Assert.assertEquals("INT", source); + return 2; + } + } + private static int timestampColumnType(String target) { switch (target) { case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv new file mode 100644 index 000000000..89f5b126d --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv @@ -0,0 +1,31 @@ +# case_id input_type input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +dec8_byte_pos BYTE 9 DECIMAL8 2 1 VALUE 000000000000005a - - - 9.0 +dec8_short_neg SHORT -9 DECIMAL8 2 1 VALUE ffffffffffffffa6 - - - -9.0 +dec8_int_pos INT 5 DECIMAL8 2 1 VALUE 0000000000000032 - - - 5.0 +dec8_int_null INT -2147483648 DECIMAL8 2 1 NULL - - - - NULL +dec16_byte_pos BYTE 99 DECIMAL16 4 2 VALUE 00000000000026ac - - - 99.00 +dec16_short_neg SHORT -99 DECIMAL16 4 2 VALUE ffffffffffffd954 - - - -99.00 +dec16_int_pos INT 42 DECIMAL16 4 2 VALUE 0000000000001068 - - - 42.00 +dec16_int_null INT -2147483648 DECIMAL16 4 2 NULL - - - - NULL +dec32_byte_min BYTE -128 DECIMAL32 9 3 VALUE fffffffffffe0c00 - - - -128.000 +dec32_short_max SHORT 32767 DECIMAL32 9 3 VALUE 0000000001f3fc18 - - - 32767.000 +dec32_int_neg INT -999999 DECIMAL32 9 3 VALUE ffffffffc46539e8 - - - -999999.000 +dec32_int_null INT -2147483648 DECIMAL32 9 3 NULL - - - - NULL +dec64_byte_max BYTE 127 DECIMAL64 18 2 VALUE 000000000000319c - - - 127.00 +dec64_short_min SHORT -32768 DECIMAL64 18 2 VALUE ffffffffffce0000 - - - -32768.00 +dec64_int_max INT 2147483647 DECIMAL64 18 2 VALUE 00000031ffffff9c - - - 2147483647.00 +dec64_int_null INT -2147483648 DECIMAL64 18 2 NULL - - - - NULL +dec128_byte_min BYTE -128 DECIMAL128 38 20 VALUE 1c50e94e78000000 fffffffffffffd4a - - -128.00000000000000000000 +dec128_short_max SHORT 32767 DECIMAL128 38 20 VALUE 434f535a9cf00000 000000000002b5de - - 32767.00000000000000000000 +dec128_int_min_plus_one INT -2147483647 DECIMAL128 38 20 VALUE ba3f5e2d63100000 fffffffd4a1c50ee - - -2147483647.00000000000000000000 +dec128_int_null INT -2147483648 DECIMAL128 38 20 NULL - - - - NULL +dec256_byte_max BYTE 127 DECIMAL256 76 57 VALUE b600000000000000 12e7deee9f226721 3b75de2e84789384 0000000000000014 127.000000000000000000000000000000000000000000000000000000000 +dec256_short_min SHORT -32768 DECIMAL256 76 57 VALUE 0000000000000000 1a55bcda92be5b00 c1a51bb2ed470a01 ffffffffffffeb9b -32768.000000000000000000000000000000000000000000000000000000000 +dec256_int_max INT 2147483647 DECIMAL256 76 57 VALUE b600000000000000 5727a1ed1eb5257c bb8496032d64c038 0000000014643e5a 2147483647.000000000000000000000000000000000000000000000000000000000 +dec256_int_null INT -2147483648 DECIMAL256 76 57 NULL - - - - NULL +byte_min_scale_zero BYTE -128 DECIMAL16 3 0 VALUE ffffffffffffff80 - - - -128 +short_min_scale_zero SHORT -32768 DECIMAL32 5 0 VALUE ffffffffffff8000 - - - -32768 +int_min_plus_one_scale_zero INT -2147483647 DECIMAL64 10 0 VALUE ffffffff80000001 - - - -2147483647 +byte_precision_overflow BYTE 10 DECIMAL8 2 1 INVALID - - - - - +short_precision_overflow SHORT 100 DECIMAL16 4 2 INVALID - - - - - +int_precision_overflow INT 1000000 DECIMAL32 9 3 INVALID - - - - - diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 65f2c03b1..41a9e584b 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7529,3 +7529,67 @@ already supported target-native VARCHAR or SYMBOL representation before the frame is persisted. Exact public wire tests prove those bytes, while existing schema-frame replay and the established long-to-text persistence coverage remain green. A setter-specific replay test would repeat unchanged persistence code. + +## D063 — Small integers share the exact integer-decimal tail + +Status: iteration 2.42 implemented and locally validated, 2026-09-14. Design +revision 65. The pre-iteration baseline is client `3acdceadbd` and server +`9bfa31a2fc`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +BYTE, SHORT and INT setters now accept DECIMAL8, DECIMAL16, DECIMAL32, +DECIMAL64, DECIMAL128 and DECIMAL256 targets in schema mode. Each non-null +source is an exact scale-zero integer. The binding passes its widened value to +one shared tail that applies the target's declared scale, checks its declared +precision and appends the matching target storage width. `Integer.MIN_VALUE` +retains its source-null meaning and becomes a scaled bitmap NULL. BYTE and SHORT +have no source-null sentinel. Missing columns still infer their native integer +types; legacy mode is unchanged. + +The current server source supports the same rule. `QwpWalAppender` dispatches +BYTE, SHORT, INT and LONG cursors to the small, 64-bit, 128-bit or 256-bit +decimal target appenders. `WalColumnarRowAppender` reads the integer through +`getLong()`, assigns scale zero, performs exact rescaling, validates precision +and storage, and writes the target decimal. The client therefore extracts the +existing LONG decimal branch into `appendIntegerDecimal` and calls it from all +four integer sources. Numeric and temporal target branches retain their earlier +short-circuits; no conversion registry, retained decimal object, per-value +allocation, protocol field or server production change was added. This is +source-level hot-path reasoning, not a new performance claim. + +Client and server consume the same 30-case corpus, SHA-256 +`e3ffe93367c2a6e65d1eeac821b3ccd798d88a9a4a7e37dfaaf9548a89d17ddf`. +It covers every one of the 18 source/target-width pairs, INT null at every +width, positive and negative scale conversion, source extrema and precision +overflow. Component tests assert the exact target wire type, scale byte, bitmap +and raw 64-bit limbs. Public socket tests additionally cover parameter rejection, +precision failure rollback, A/error/C row recovery and a DECIMAL(3,0) block +remaining pinned before a DECIMAL(20,5) schema generation. + +The real-server corpus test sends every valid vector through public Sender +setters, waits for the acknowledged sequence, drains WAL and asserts exact SQL +values and nulls. A second public-Sender test proves a partially written row is +discarded after an INT-to-UUID local error while completed decimal rows survive. +The focused server gate also runs the existing decimal aggregate and variant +tests: four tests total, all green. + +Expanded client validation passes 323 tests with zero failures, errors or skips. +Packaged-JAR verification passes 48 selected unit tests and both packaging +integration tests. The released-binary matrix passes current client to QuestDB +10.0.1 in legacy mode, client 1.3.9 to current server in legacy mode, and current +client to current server in schema mode. The matrix run used +`/mnt/pcie5/qwp-small-integer-decimal-compat/run.0W5Ji1`; all other scratch roots +also remain under `/mnt/pcie5`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 34 errors, down from 36 errors at D062. Both decimal aggregate +tests are now green. The open intentional implementation entries are +INT-to-IPv4, STRING-to-DATE and DOUBLE arrays. STRING/VARCHAR aggregates still +stop at the separately unresolved CHAR-to-text contract; other errors are +legacy tests whose expected server-side rejection is now an early local error. + +No new store-and-forward test was added. The changed code selects target-native +decimal bytes before persistence and reuses the already exercised decimal +buffer representation. Exact public wire tests prove those bytes; the existing +schema-frame replay suite remains green. A small-integer-specific recovery test +would repeat unchanged persistence code. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 85c5e2adf..aed0dcb13 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 64. Scope: QWP v1 over +Status: implemented on the development branch, revision 65. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.40. Iteration -2.41 (small-integer text targets) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.41. Iteration +2.42 (small-integer decimal targets) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,10 +39,10 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.41 adds BYTE, SHORT and INT input to STRING, -VARCHAR and SYMBOL targets. Its pre-iteration baseline is client `853c3292c1` -and server `85f24785d3`, whose submodule pins that exact client revision. The -small-integer text slice is the current locally validated increment. +Current checkpoint: iteration 2.42 adds BYTE, SHORT and INT input to all six +decimal target widths. Its pre-iteration baseline is client `3acdceadbd` and +server `9bfa31a2fc`, whose submodule pins that exact client revision. The +small-integer decimal slice is the current locally validated increment. The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: @@ -167,6 +167,18 @@ corpus, exact-wire tests, generation rebind, legacy isolation, public-Sender real-server ingestion and A/error/C rollback cover the slice. D062 records the decision and evidence. +BYTE, SHORT and INT inputs now select DECIMAL8, DECIMAL16, DECIMAL32, +DECIMAL64, DECIMAL128 and DECIMAL256 target representations directly. All three +sources have scale zero, then reuse the established exact integer-decimal tail: +the value is rescaled to the target scale, checked against the declared +precision and appended in the target storage width. `Integer.MIN_VALUE` remains +a source NULL and records the target scale with a bitmap NULL; BYTE and SHORT +have no source-null sentinel. One shared helper also replaces the identical +LONG decimal tail without changing its behavior. Shared exact-wire and +real-server vectors, precision rejection, partial-row rollback and a decimal +precision/scale schema rebind cover the slice. D063 records the decision and +evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -943,8 +955,8 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.41 on 2026-09-14. The pre-iteration baseline is -client `853c3292c1` and server `85f24785d3`, whose submodule pins that exact +refreshed through iteration 2.42 on 2026-09-14. The pre-iteration baseline is +client `3acdceadbd` and server `9bfa31a2fc`, whose submodule pins that exact client revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; @@ -975,8 +987,8 @@ non-null conversion. Known compatibility exceptions above still apply. | Public input (source wire type) | Implemented targets | Missing server-accepted targets | | --- | --- | --- | | `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | -| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps, text, SYMBOL | Decimals | -| `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL | Decimals, IPv4 | +| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps, text, SYMBOL, decimals | None | +| `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL, decimals | IPv4 | | `longColumn` (LONG) | Numeric, text, SYMBOL, DATE, timestamps, decimals | None | | `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | | `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | @@ -1698,6 +1710,12 @@ They do not change the compatibility contract. INT widen losslessly to long, so use the established reusable long sink and direct STRING/VARCHAR or SYMBOL append paths. Do not add source-width formatters, boxed values, per-row strings or a text conversion registry. +24. **Reuse one scale-zero integer decimal tail.** BYTE, SHORT, INT and LONG + differ in input width and null sentinel, but every non-null value begins as + an exact scale-zero integer. Pass that value and its source-null decision to + the established rescale, precision-check and target-width append logic. Do + not add one encoder per source width, a decimal conversion registry or + retained decimal objects. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1713,7 +1731,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.41 +### Plan after iteration 2.42 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1751,16 +1769,21 @@ and TIMESTAMP_NS target representations, and LONG selects DATE. Source values remain raw counts in the target unit; no unit conversion or new temporal abstraction was added. -Iteration 2.41 is complete locally: BYTE, SHORT and INT select STRING, VARCHAR +Iteration 2.41 is committed: BYTE, SHORT and INT select STRING, VARCHAR and SYMBOL target representations through the existing reusable integer formatter and direct text/symbol append paths. The real-server conversion diagnostic improves by exactly one error because the SYMBOL aggregate is now green; STRING and VARCHAR continue to the independent CHAR-to-text gap. -The next bounded slice is BYTE, SHORT and INT to decimal targets. Reuse the -established exact decimal append tail rather than add another conversion path. -Then implement the separate INT-to-IPv4 null translation and STRING-to-DATE -parsing. DOUBLE arrays follow only after rank metadata is proven end to end. +Iteration 2.42 is complete locally: BYTE, SHORT and INT select every decimal +target width through the established exact integer-decimal tail. The target's +declared precision and scale control rescaling and storage; no new decimal +representation or server production code was added. + +The next bounded slice is INT-to-IPv4. Treat it as target representation +selection plus the server's specific INT-null translation, not a general +integer/address conversion. Then implement STRING-to-DATE parsing. DOUBLE +arrays follow only after rank metadata is proven end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From 19557e2f87d4f7df8a29f206f62fa42d86579c0d Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 18:42:53 +0200 Subject: [PATCH 12/51] feat(qwp): support int to IPv4 conversion --- .../qwp/protocol/QwpSchemaBinding.java | 8 ++ .../QwpSchemaSenderIntegrationTest.java | 82 +++++++++++++++++++ .../qwp/protocol/QwpSchemaBindingTest.java | 46 ++++++++++- design/schema-aware-sender-journal.md | 60 ++++++++++++++ design/schema-aware-sender.md | 48 +++++++---- 5 files changed, 227 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index a21dccfbb..68fc8f687 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -1131,6 +1131,7 @@ private QwpSchemaBinding integerColumn( if (!isNumericTarget(targetType) && !isTextTarget(targetType) && !ColumnType.isDecimal(targetType) + && !(inferredType == ColumnType.INT && targetType == ColumnType.IPv4) && targetType != ColumnType.DATE && targetType != ColumnType.TIMESTAMP_MICRO && targetType != ColumnType.TIMESTAMP_NANO) { @@ -1160,6 +1161,13 @@ private QwpSchemaBinding integerColumn( case ColumnType.INT: column.addInt((int) value); break; + case ColumnType.IPv4: + if (value == Numbers.IPv4_NULL) { + column.addNull(); + } else { + column.addIPv4((int) value); + } + break; case ColumnType.LONG: case ColumnType.DATE: case ColumnType.TIMESTAMP_MICRO: diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 9796525d8..731c65851 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -319,6 +319,60 @@ public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Excep }); } + @Test + public void testIntToIpv4GenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1101, 1111, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").intColumn("value", 1).atNow(); + + handler.targetType = ColumnType.IPv4; + handler.version = 1112; + sender.table("events"); + try { + sender.intColumn("value", 1_000); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.intColumn("value", 1_000).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).twoIntDecimalAndIpv4Blocks( + "events", 1101, 1111, 1112); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1121, 1131, ColumnType.IPv4); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 0).atNow(); + + sender.intColumn("value", 0x0a000001); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.intColumn("value", 1).atNow(); + sender.intColumn("value", -1).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).intToIpv4Table("events", 1121, 1131); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + }); + } + @Test public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { assertMemoryLeak(() -> { @@ -2243,6 +2297,18 @@ private void ipv4Table(String table, int tableId, long version, int targetType, eof(); } + private void intToIpv4Table(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, u8()); + Assert.assertEquals(3, u8()); + Assert.assertEquals(1, in.getInt()); + Assert.assertEquals(-1, in.getInt()); + eof(); + } + private void long256TargetTable( String table, int tableId, @@ -2923,6 +2989,22 @@ private void twoSmallIntegerDecimalBlocks( eof(); } + private void twoIntDecimalAndIpv4Blocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1_000, in.getInt()); + eof(); + } + private void twoUuidAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { messageHeader(2); Assert.assertEquals(0, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index c4467e4d2..0401a9937 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -820,6 +820,39 @@ public void testIntMinIsNullWithCompanionMissingRowsAcrossNumericTargets() throw }); } + @Test + public void testIntToIpv4UsesTargetNativeWireAndNulls() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ip", ColumnType.IPv4)); + rows.intColumn("ip", Integer.MIN_VALUE); + buffer.nextRow(); + rows.intColumn("ip", 0); + buffer.nextRow(); + rows.intColumn("ip", 1); + buffer.nextRow(); + rows.intColumn("ip", Integer.MIN_VALUE + 1); + buffer.nextRow(); + rows.intColumn("ip", Integer.MAX_VALUE); + buffer.nextRow(); + rows.intColumn("ip", -1); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 7, QwpConstants.TYPE_IPv4); + Assert.assertEquals("NULL bitmap must be present", 1, reader.byteValue()); + Assert.assertEquals("INT null, IPv4 zero and omission must be NULL", 0x43, reader.byteValue()); + Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(Integer.MIN_VALUE + 1, reader.intValue()); + Assert.assertEquals(Integer.MAX_VALUE, reader.intValue()); + Assert.assertEquals(-1, reader.intValue()); + Assert.assertEquals(size, reader.position()); + } + }); + } + @Test public void testSmallIntegerMissingSchemaUsesNativeWireTypes() throws Exception { assertMemoryLeak(() -> { @@ -847,13 +880,21 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { QwpSchemaBinding rows = rows(buffer, column("uuid", ColumnType.UUID), + column("ip", ColumnType.IPv4), column("future", ColumnType.INT, new byte[]{1}), column("future_date", ColumnType.DATE, new byte[]{1}), column("future_text", ColumnType.VARCHAR, new byte[]{1}), - column("future_decimal", ColumnType.getDecimalType(18, 2), new byte[]{1})); + column("future_decimal", ColumnType.getDecimalType(18, 2), new byte[]{1}), + column("future_ipv4", ColumnType.IPv4, new byte[]{1})); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.byteColumn("uuid", (byte) 1)); rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("ip", (byte) 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("ip", (short) 2)); + rollbackCurrentRow(buffer); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.shortColumn("future", (short) 2)); rollbackCurrentRow(buffer); @@ -866,6 +907,9 @@ public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.shortColumn("future_decimal", (short) 5)); rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("future_ipv4", 6)); + rollbackCurrentRow(buffer); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.intColumn("uuid", Integer.MIN_VALUE)); } diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 41a9e584b..9c4dcf3b3 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7593,3 +7593,63 @@ decimal bytes before persistence and reuses the already exercised decimal buffer representation. Exact public wire tests prove those bytes; the existing schema-frame replay suite remains green. A small-integer-specific recovery test would repeat unchanged persistence code. + +## D064 — INT-to-IPv4 translates both source and target null sentinels + +Status: iteration 2.43 implemented and locally validated, 2026-09-14. Design +revision 66. The pre-iteration baseline is client `1630c04775` and server +`50d487e8b5`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +`intColumn` now accepts an IPv4 target in schema mode and emits target-native +`TYPE_IPv4`. `Integer.MIN_VALUE` is the INT source null sentinel and becomes an +explicit bitmap NULL. A supplied integer zero also becomes bitmap NULL because +zero is QuestDB's IPv4 null sentinel. Every other 32-bit pattern is appended +unchanged, including negative Java integers representing addresses whose first +octet is at least 128. BYTE, SHORT and LONG remain rejected for IPv4, and a +parameterized IPv4 target remains rejected. + +This rule is cross-checked against the current server's `QwpWalAppender` IPv4 +dispatch, `QwpFixedWidthColumnCursor` null detection and +`WalColumnarRowAppender.putIntToIPv4Column`. The legacy server path reads a +TYPE_INT cursor, translates its source-null row to IPv4 zero, and copies every +other integer verbatim. The client performs the equivalent conversion before +buffering, using the existing four-byte IPv4 column. The implementation adds +one allowed-target condition and one switch arm; it adds no parser, formatter, +address object, allocation, retained state, protocol field or server production +change. Other integer target arms are unchanged. This is source-level reasoning, +not a measured throughput claim. + +The component test covers `Integer.MIN_VALUE`, zero, one, minimum-plus-one, +maximum, all-one bits and omission. It asserts the exact TYPE_IPv4 wire code, +bitmap and four stored values. Negative tests retain BYTE/SHORT rejection and +cover IPv4 parameter rejection. Public socket tests prove A/error/C rollback, +the standard one-refresh lifecycle, exact target wire data, and a DECIMAL(3,0) +block remaining pinned before a new IPv4 generation. + +The real-server E2E uses public Sender calls for both null cases and six +non-null patterns from `0.0.0.1` through `255.255.255.255`, plus omission. It +waits for the ACK, drains WAL and checks exact rendered IPv4 values and marker +rows; a partial `failed-B` row is absent after an INT-to-UUID local error. The +focused gate also runs the existing INT-null IPv4 test: two tests total, both +green. Its stale comment was corrected because the current client now exercises +target-native schema encoding rather than the server's legacy TYPE_INT arm. + +Focused client validation passes 116 tests. Expanded client validation passes +326 tests, all with zero failures, errors or skips. Packaged-JAR verification +passes 49 selected unit tests and both packaging integration tests. The +released-binary matrix passes all three old/new combinations with current JAR +SHA-256 `82140012a88ee02b4aff5e7554a722f9f8a55b1eb83fb3763bd027a9cbc1469d`; +the run used `/mnt/pcie5/qwp-int-ipv4-compat/run.CKpGSZ`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 33 errors, down from 34 errors at D063. The INT-to-IPv4 test is now +green. The only intentional conversion implementations left are STRING-to-DATE +and DOUBLE arrays; the separately unresolved and legacy-expectation cases remain +outside this slice. + +No new store-and-forward test was added. The changed code selects the existing +target-native IPv4 representation before persistence. Exact public wire tests +prove those bytes, while the unchanged schema-frame replay suite remains green. +An INT-specific recovery test would repeat the already covered opaque-frame +persistence contract. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index aed0dcb13..9d935ab69 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 65. Scope: QWP v1 over +Status: implemented on the development branch, revision 66. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.41. Iteration -2.42 (small-integer decimal targets) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.42. Iteration +2.43 (INT-to-IPv4) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,10 +39,10 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.42 adds BYTE, SHORT and INT input to all six -decimal target widths. Its pre-iteration baseline is client `3acdceadbd` and -server `9bfa31a2fc`, whose submodule pins that exact client revision. The -small-integer decimal slice is the current locally validated increment. +Current checkpoint: iteration 2.43 adds INT input to an IPv4 target. Its +pre-iteration baseline is client `1630c04775` and server `50d487e8b5`, whose +submodule pins that exact client revision. INT-to-IPv4 is the current locally +validated increment. The wider conversion inventory below is not complete; this is not release acceptance. The standing compatibility gate now runs three real process combinations: @@ -179,6 +179,15 @@ real-server vectors, precision rejection, partial-row rollback and a decimal precision/scale schema rebind cover the slice. D063 records the decision and evidence. +INT input now selects target-native IPv4 wire representation. The input's +`Integer.MIN_VALUE` sentinel becomes an explicit bitmap NULL; the integer value +zero also becomes NULL because zero is the target IPv4 sentinel. Every other +32-bit pattern is appended unchanged. BYTE, SHORT and LONG remain unsupported +for IPv4, so this is not a general integer/address conversion. Exact-wire tests, +public-Sender ingestion, partial-row rollback, parameter rejection and a +DECIMAL-to-IPv4 schema rebind cover the slice. D064 records the decision and +evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -955,8 +964,8 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.42 on 2026-09-14. The pre-iteration baseline is -client `3acdceadbd` and server `9bfa31a2fc`, whose submodule pins that exact +refreshed through iteration 2.43 on 2026-09-14. The pre-iteration baseline is +client `1630c04775` and server `50d487e8b5`, whose submodule pins that exact client revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; @@ -988,7 +997,7 @@ non-null conversion. Known compatibility exceptions above still apply. | --- | --- | --- | | `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | | `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps, text, SYMBOL, decimals | None | -| `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL, decimals | IPv4 | +| `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL, decimals, IPv4 | None | | `longColumn` (LONG) | Numeric, text, SYMBOL, DATE, timestamps, decimals | None | | `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | | `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | @@ -1716,6 +1725,11 @@ They do not change the compatibility contract. the established rescale, precision-check and target-width append logic. Do not add one encoder per source width, a decimal conversion registry or retained decimal objects. +25. **Treat INT-to-IPv4 as sentinel-aware representation selection.** Both + types carry one 32-bit value, but their null sentinels differ. Translate + INT null and IPv4 zero to the target bitmap, then append every other bit + pattern through the existing IPv4 column. Do not add address parsing, + widening, an address object or a general numeric-to-IPv4 rule. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1731,7 +1745,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.42 +### Plan after iteration 2.43 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1775,15 +1789,17 @@ formatter and direct text/symbol append paths. The real-server conversion diagnostic improves by exactly one error because the SYMBOL aggregate is now green; STRING and VARCHAR continue to the independent CHAR-to-text gap. -Iteration 2.42 is complete locally: BYTE, SHORT and INT select every decimal +Iteration 2.42 is committed: BYTE, SHORT and INT select every decimal target width through the established exact integer-decimal tail. The target's declared precision and scale control rescaling and storage; no new decimal representation or server production code was added. -The next bounded slice is INT-to-IPv4. Treat it as target representation -selection plus the server's specific INT-null translation, not a general -integer/address conversion. Then implement STRING-to-DATE parsing. DOUBLE -arrays follow only after rank metadata is proven end to end. +Iteration 2.43 is complete locally: INT selects target-native IPv4 storage, +normalizing the INT and IPv4 null sentinels without adding parsing, allocation +or a general numeric/address conversion. + +The next bounded slice is STRING-to-DATE parsing. DOUBLE arrays follow only +after rank metadata is proven end to end. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From 47a37a6f31ba795332797e85181373fbba296c6e Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 19:14:34 +0200 Subject: [PATCH 13/51] feat(qwp): support string to date conversion --- .../qwp/protocol/QwpSchemaBinding.java | 21 ++ .../protocol/QwpSchemaTimestampParser.java | 227 ++++++++++++- .../QwpSchemaSenderIntegrationTest.java | 79 +++++ .../QwpSchemaBindingStringDateTest.java | 312 ++++++++++++++++++ .../QwpSchemaBindingStringNumericTest.java | 2 +- .../client/cutlass/qwp/string-to-date.tsv | 54 +++ design/schema-aware-sender-journal.md | 65 ++++ design/schema-aware-sender.md | 58 +++- 8 files changed, 799 insertions(+), 19 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java create mode 100644 core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 68fc8f687..da495a2af 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -616,6 +616,9 @@ public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { case ColumnType.CHAR: column.addShort((short) stringToChar(value)); break; + case ColumnType.DATE: + appendStringDate(column, name, value, targetType); + break; case ColumnType.TIMESTAMP_MICRO: case ColumnType.TIMESTAMP_NANO: appendStringTimestamp(column, name, value, targetType); @@ -1414,6 +1417,24 @@ private void appendStringGeoHash( } } + private void appendStringDate( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + long millis = QwpSchemaTimestampParser.parseDate(value); + if (millis == Long.MIN_VALUE) { + column.addNull(); + } else { + column.addLong(millis); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid DATE text"); + } + } + private void appendStringTimestamp( QwpTableBuffer.ColumnBuffer column, CharSequence name, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java index a3dcbacf1..0e91217ea 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java @@ -26,13 +26,19 @@ import java.util.Set; /** - * Source-faithful parser for the eleven formats accepted by QWP STRING-to-timestamp conversion. - * The grammar and computation mirror the fixed formats in the server's - * {@code MicrosFormatUtils} and {@code GenericMicrosFormat}; this is deliberately - * not a general date-format compiler. + * Source-faithful parser for the fixed formats accepted by QWP STRING-to-timestamp + * and STRING-to-DATE conversion. The grammar and computation mirror the server's + * {@code MicrosFormatUtils}, {@code DateFormatUtils} and generic format parsers; + * this is deliberately not a general date-format compiler. */ final class QwpSchemaTimestampParser { private static final long DAY = 86_400_000_000L; + private static final long DATE_DAY = 86_400_000L; + private static final long DATE_HOUR = 3_600_000L; + private static final long DATE_MINUTE = 60_000L; + private static final long DATE_SECOND = 1_000L; + private static final long[] DATE_MONTH_COMMON = dateMonthStarts(false); + private static final long[] DATE_MONTH_LEAP = dateMonthStarts(true); private static final int DAYS_0000_TO_1970 = 719527; private static final long HOUR = 3_600_000_000L; private static final long MINUTE = 60_000_000L; @@ -58,6 +64,82 @@ static long parse(CharSequence value) throws NumericException { throw failure; } + static long parseDate(CharSequence value) throws NumericException { + NumericException failure = NumericException.instance(); + try { + return parsePgDate(value); + } catch (NumericException ignored) { + } + try { + return parseUtcDate(value); + } catch (NumericException ignored) { + } + try { + return Numbers.parseLong(value); + } catch (NumericException ignored) { + } + throw failure; + } + + private static long parsePgDate(CharSequence value) throws NumericException { + int n = value.length(); + int dash = indexOf(value, '-', n > 0 && value.charAt(0) == '-' ? 1 : 0, n); + if (dash < 1) { + throw NumericException.instance(); + } + int year = greedyInt(value, 0, dash); + if (dash == 2) { + year += REFERENCE_CENTURY; + } + int pos = dash + 1; + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return computeDate(year, month, day, 0, 0, 0, 0, null); + } + require(value, pos++, n, ' '); + int tail = pos; + try { + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + int millis = 0; + if (pos < n && value.charAt(pos) == '.') { + long parsed = greedyMillis(value, pos + 1, n); + millis = (int) parsed; + pos += 1 + (int) (parsed >>> 32); + } + return computeDate(year, month, day, hour, minute, second, millis, + ZoneMatcher.parse(value, pos, n)); + } catch (NumericException ignored) { + return computeDate(year, month, day, 0, 0, 0, 0, + ZoneMatcher.parse(value, tail, n)); + } + } + + private static long parseUtcDate(CharSequence value) throws NumericException { + int n = value.length(); + int pos = 0; + int year = fixedInt(value, pos, pos += 4, n); + require(value, pos++, n, '-'); + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, 'T'); + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '.'); + int millis = fixedInt(value, pos, pos += 3, n); + return computeDate(year, month, day, hour, minute, second, millis, + ZoneMatcher.parse(value, pos, n)); + } + private static long parseIso(CharSequence value) throws NumericException { int n = value.length(); Parsed p = new Parsed(); @@ -204,6 +286,27 @@ private static long compute(Parsed p, ZoneMatcher.Zone zone) throws NumericExcep return zone == null ? out : out - zone.offset(out, p.year); } + private static long computeDate( + int year, + int month, + int day, + int hour, + int minute, + int second, + int millis, + ZoneMatcher.Zone zone + ) throws NumericException { + boolean leap = CommonUtils.isLeapYear(year); + if (month < 1 || month > 12 || day < 1 || day > CommonUtils.getDaysPerMonth(month, leap) + || hour < 0 || hour > 24 || minute < 0 || minute > 59 + || second < 0 || second > 59) { + throw NumericException.instance(); + } + long out = dateYearMillis(year, leap) + dateMonthMillis(month, leap) + (day - 1L) * DATE_DAY + + (hour % 24L) * DATE_HOUR + minute * DATE_MINUTE + second * DATE_SECOND + millis; + return zone == null ? out : out - zone.offsetMillis(out, year); + } + private static int parseFixedYear(CharSequence s, int pos, int hi, Parsed p) throws NumericException { int width = pos < hi && s.charAt(pos) == '-' ? 5 : 4; p.year = width == 5 @@ -274,6 +377,56 @@ private static long[] monthStarts(boolean leap) { return result; } + private static long[] dateMonthStarts(boolean leap) { + long[] result = new long[12]; + for (int i = 1; i < 12; i++) { + result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap) * DATE_DAY; + } + return result; + } + + private static long dateMonthMillis(int month, boolean leap) { + return (leap ? DATE_MONTH_LEAP : DATE_MONTH_COMMON)[month - 1]; + } + + private static long dateYearMillis(int year, boolean leap) { + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + return (year * 365L + leapYears - DAYS_0000_TO_1970) * DATE_DAY; + } + + private static int dateDayOfWeek(long millis) { + long d; + if (millis > -1) { + d = millis / DATE_DAY; + } else { + d = (millis - (DATE_DAY - 1)) / DATE_DAY; + if (d < -3) { + return 7 + (int) ((d + 4) % 7); + } + } + return 1 + (int) ((d + 3) % 7); + } + + private static long nextDateOrSame(long millis, int dow) { + int current = dateDayOfWeek(millis); + return current <= dow ? millis + (dow - current) * DATE_DAY + : millis + (7 - current + dow) * DATE_DAY; + } + + private static long previousDateOrSame(long millis, int dow) { + int current = dateDayOfWeek(millis); + return current >= dow ? millis - (current - dow) * DATE_DAY + : millis - (7 + current - dow) * DATE_DAY; + } + private static long monthMicros(int month, boolean leap) { return (leap ? MONTH_LEAP : MONTH_COMMON)[month - 1]; } @@ -561,6 +714,36 @@ private long offset(long epoch, int year) { return rules.getOffset(instant).getTotalSeconds() * SECOND; } + private long offsetMillis(long epoch, int year) throws NumericException { + if (fixedOffset != Long.MIN_VALUE) { + return fixedOffset / 1000L; + } + long cutoffMillis = cutoff == Long.MIN_VALUE ? Long.MIN_VALUE : cutoff / 1000L; + if (!recurring.isEmpty() && epoch > cutoffMillis) { + // The server's recurring-rule cache rejects negative year keys. This can + // only be reached after full-width DATE arithmetic wraps past its cutoff; + // reject it as an invalid value instead of silently accepting bytes that + // the server's STRING conversion cannot produce. + if (year < 0) { + throw NumericException.instance(); + } + long after = 0; + for (ZoneOffsetTransitionRule rule : recurring) { + long transition = transitionEpochMillis(rule, year); + long before = rule.getOffsetBefore().getTotalSeconds() * DATE_SECOND; + if (epoch < transition) { + return before; + } + after = rule.getOffsetAfter().getTotalSeconds() * DATE_SECOND; + } + return after; + } + if (epoch > cutoffMillis) { + return lastWall / 1000L; + } + return rules.getOffset(Instant.ofEpochMilli(epoch)).getTotalSeconds() * DATE_SECOND; + } + private static long transitionEpoch(ZoneOffsetTransitionRule rule, int year) { boolean leap = CommonUtils.isLeapYear(year); int month = rule.getMonth().getValue(); @@ -594,6 +777,42 @@ private static long transitionEpoch(ZoneOffsetTransitionRule rule, int year) { } return timestamp - before * SECOND; } + + private static long transitionEpochMillis(ZoneOffsetTransitionRule rule, int year) { + boolean leap = CommonUtils.isLeapYear(year); + int month = rule.getMonth().getValue(); + int dom = rule.getDayOfMonthIndicator(); + int dow = rule.getDayOfWeek() == null ? -1 : rule.getDayOfWeek().getValue(); + long timestamp; + if (dom < 0) { + timestamp = dateYearMillis(year, leap) + dateMonthMillis(month, leap) + + (CommonUtils.getDaysPerMonth(month, leap) + dom) * DATE_DAY + + rule.getLocalTime().getHour() * DATE_HOUR + + rule.getLocalTime().getMinute() * DATE_MINUTE + + rule.getLocalTime().getSecond() * DATE_SECOND; + if (dow > -1) { + timestamp = previousDateOrSame(timestamp, dow); + } + } else { + timestamp = dateYearMillis(year, leap) + dateMonthMillis(month, leap) + (dom - 1L) * DATE_DAY + + rule.getLocalTime().getHour() * DATE_HOUR + + rule.getLocalTime().getMinute() * DATE_MINUTE + + rule.getLocalTime().getSecond() * DATE_SECOND; + if (dow > -1) { + timestamp = nextDateOrSame(timestamp, dow); + } + } + if (rule.isMidnightEndOfDay()) { + timestamp += DATE_DAY; + } + int before = rule.getOffsetBefore().getTotalSeconds(); + if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.UTC) { + timestamp += before * DATE_SECOND; + } else if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.STANDARD) { + timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * DATE_SECOND; + } + return timestamp - before * DATE_SECOND; + } } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 731c65851..e8e1d93ce 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -871,6 +871,55 @@ public void testStringToTimestampTargetsUsePinnedWireAndRollback() throws Except }); } + @Test + public void testStringToDateUsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1141, 1151, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "not-a-date"); + Assert.fail("expected DATE parse error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.stringColumn("value", "1969-12-31").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .stringDateTable("events", 1141, 1151, -86_400_000L); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testDateGenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1161, 1171, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "1970-01-01").atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 1172; + sender.table("events"); + try { + sender.stringColumn("value", "native-a"); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.stringColumn("value", "native-a").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .twoDateAndVarcharBlocks("events", 1161, 1171, 1172); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + @Test public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { assertMemoryLeak(() -> { @@ -2842,6 +2891,36 @@ private void timestampTextTable( eof(); } + private void stringDateTable( + String table, int tableId, long version, long expectedValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(expectedValue, i64()); + eof(); + } + + private void twoDateAndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + eof(); + } + private void charTable(String table, int tableId, long version, char first, char second) { messageHeader(1); Assert.assertEquals(0, varint()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java new file mode 100644 index 000000000..027d592dd --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java @@ -0,0 +1,312 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringDateTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-date.tsv"; + + @Test + public void testCorpusUsesExactDateWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringDateTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("case_id\tinput_kind\tinput\toutcome\texpected_raw\tfeature", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + CharSequence value = input(fields[1], fields[2]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DATE)); + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + Assert.assertEquals(0, buffer.getRowCount()); + } else { + Assert.assertTrue(line, "VALID".equals(fields[3]) || "NULL".equals(fields[3])); + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableReader(encoder, size, 1, "value", QwpConstants.TYPE_DATE); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(53, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.DATE), column("bad", ColumnType.UUID), column("c", ColumnType.DATE)); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-02"); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-03"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "1969-12-31"); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 4, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(13, reader.u8()); + Assert.assertEquals(86_400_000L, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(-86_400_000L, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "1970-01-01 00:00:00.001UTC"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + Reader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testNativeInferenceParameterizedAndDesignatedBehavior() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "1970-01-01"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DATE, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "1970-01-01")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("value", ColumnType.DATE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "1970-01-01")); + } + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type) + .putShort((short) params.length).put(params).array(); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static CharSequence input(String kind, String value) { + Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); + if ("".equals(value)) { + return null; + } + if ("".equals(value)) { + return ""; + } + if ("TEXT".equals(kind)) { + return value; + } + String[] units = value.split(","); + char[] chars = new char[units.length]; + for (int i = 0; i < units.length; i++) { + chars[i] = (char) Integer.parseInt(units[i], 16); + } + return new String(chars); + } + + private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + Reader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position <= limit - Integer.BYTES); + int value = Unsafe.getUnsafe().getInt(address + position); + position += Integer.BYTES; + return value; + } + + private long i64() { + Assert.assertTrue(position <= limit - Long.BYTES); + long value = Unsafe.getUnsafe().getLong(address + position); + position += Long.BYTES; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + while (true) { + int value = u8(); + result |= (value & 0x7f) << shift; + if ((value & 0x80) == 0) { + return result; + } + shift += 7; + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java index dc848d7e7..51747d5b5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java @@ -196,7 +196,7 @@ public void testNullAndParsedSentinelsHaveDistinctWirePresence() throws Exceptio public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { QwpSchemaBinding binding = binding(buffer, -1, - column("bool", ColumnType.BOOLEAN), column("date", ColumnType.DATE), + column("bool", ColumnType.BOOLEAN), column("date", ColumnType.DATE, new byte[]{1}), column("future", ColumnType.INT, new byte[]{1}), column("flagged", ColumnType.LONG | 0x10000)); assertSchemaError(LineSenderSchemaException.Reason.INVALID_VALUE, "bool", "BOOLEAN", () -> binding.stringColumn("bool", "2")); diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv new file mode 100644 index 000000000..d7c321ebb --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv @@ -0,0 +1,54 @@ +case_id input_kind input outcome expected_raw feature +java_null TEXT NULL java_null +empty TEXT INVALID empty +raw_zero TEXT 0 VALID 0 raw_millis +raw_negative TEXT -1 VALID -1 raw_millis +raw_suffix TEXT 1577836800000L VALID 1577836800000 raw_suffix +raw_underscore TEXT 1_000 VALID 1000 raw_underscore +raw_null_sentinel TEXT -9223372036854775808 NULL raw_null_sentinel +raw_max TEXT 9223372036854775807 VALID 9223372036854775807 raw_boundary +raw_overflow TEXT 9223372036854775808 INVALID raw_overflow +epoch_date TEXT 1970-01-01 VALID 0 pg_date +pre_epoch_date TEXT 1969-12-31 VALID -86400000 pg_date +leap_date TEXT 2000-02-29 VALID 951782400000 pg_date +non_leap_date TEXT 1900-02-29 INVALID calendar_validation +date_time_utc TEXT 2020-01-01 00:00:00UTC VALID 1577836800000 pg_datetime +date_zone_utc TEXT 2020-01-01 UTC VALID 1577836800000 pg_date_zone +fraction_one TEXT 2020-01-01 00:00:00.1UTC VALID 1577836800100 greedy_millis +fraction_two TEXT 2020-01-01 00:00:00.12UTC VALID 1577836800120 greedy_millis +fraction_three TEXT 2020-01-01 00:00:00.123UTC VALID 1577836800123 greedy_millis +fraction_negative TEXT 2020-01-01 00:00:00.-1UTC VALID 1577836799990 greedy_millis +fraction_four TEXT 2020-01-01 00:00:00.1234UTC INVALID greedy_millis +iso_z TEXT 2020-01-01T00:00:00.123Z VALID 1577836800123 utc_format +iso_named_utc TEXT 2020-01-01T00:00:00.123UTC VALID 1577836800123 utc_format +iso_offset TEXT 2020-01-01T00:00:00.123+02:30 VALID 1577827800123 utc_format +iso_without_millis TEXT 2020-01-01T00:00:00Z INVALID utc_format +iso_two_millis TEXT 2020-01-01T00:00:00.12Z INVALID utc_format +iso_four_millis TEXT 2020-01-01T00:00:00.1234Z INVALID utc_format +offset_datetime TEXT 2020-01-01 00:00:00+02:00 VALID 1577829600000 numeric_zone +offset_date TEXT 2020-01-01 +02:00 VALID 1577829600000 numeric_zone +berlin_winter TEXT 2020-01-01 12:00:00Europe/Berlin VALID 1577876400000 named_zone +berlin_summer TEXT 2020-07-01 12:00:00Europe/Berlin VALID 1593597600000 named_zone +berlin_gap TEXT 2020-03-29 02:30:00Europe/Berlin VALID 1585441800000 zone_transition +berlin_overlap TEXT 2020-10-25 02:30:00Europe/Berlin VALID 1603589400000 zone_transition +hour_24 TEXT 2020-01-01 24:59:59UTC VALID 1577840399000 hour_24_wrap +hour_25 TEXT 2020-01-01 25:00:00UTC INVALID time_validation +minute_60 TEXT 2020-01-01 00:60:00UTC INVALID time_validation +second_60 TEXT 2020-01-01 00:00:60UTC INVALID time_validation +year_zero TEXT 0000-01-01 VALID -62167219200000 proleptic_year +negative_year TEXT -0001-01-01 VALID -62198755200000 proleptic_year +max_int_year TEXT 2147483647-01-01 VALID -6019000092841406464 year_wrap +min_int_year TEXT -2147483648-01-01 VALID 6018875726867006464 year_wrap +negative_year_named_zone_wrap TEXT -1000000000-01-01 00:00:00Europe/London INVALID server_unchecked_guard +year_overflow TEXT 2147483648-01-01 INVALID year_overflow +plus_year TEXT +1-01-01 INVALID strict_year +underscore_year TEXT 2_0-01-01 INVALID strict_year +bad_month TEXT 2020-13-01 INVALID calendar_validation +bad_day TEXT 2020-02-30 INVALID calendar_validation +missing_zone TEXT 2020-01-01 00:00:00 INVALID zone_required +unknown_zone TEXT 2020-01-01 Mars/Base INVALID zone_validation +trailing_date_space TEXT 2020-01-01 INVALID no_tail +trailing_zone_space TEXT 2020-01-01 UTC INVALID no_tail +raw_trailing_text TEXT 1_000x INVALID no_tail +unicode_digits UTF16_HEX ff11,ff12,ff13 INVALID ascii_only +plain_text TEXT not-a-date INVALID invalid_text diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 9c4dcf3b3..90f8eecf7 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7653,3 +7653,68 @@ target-native IPv4 representation before persistence. Exact public wire tests prove those bytes, while the unchanged schema-frame replay suite remains green. An INT-specific recovery test would repeat the already covered opaque-frame persistence contract. + +## D065 — STRING-to-DATE mirrors the fixed server grammar before buffering + +Status: iteration 2.44 implemented and locally validated, 2026-09-14. Design +revision 67. The pre-iteration baseline is client `19557e2f` and server +`5338ac0845`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +`stringColumn` now accepts an ordinary DATE target in schema mode. It attempts +the same formats, in the same order, as the current server's +`DateFormatUtils.parseDate`: PostgreSQL date-time, date-only, date plus zone, +date-time with milliseconds plus zone, UTC date-time with exactly three +fractional digits, then a raw signed epoch-millisecond long. Successful values +are appended as target-native `TYPE_DATE`; `Long.MIN_VALUE` becomes an explicit +bitmap NULL. Missing columns still infer VARCHAR. A parameterized DATE and a +designated DATE remain rejected. + +The implementation extends the existing fixed timestamp parser with DATE's +five parse attempts and millisecond calendar/timezone arithmetic. It reuses the +same immutable JDK-derived timezone metadata and performs no per-value date +object allocation. It does not add a general format compiler, converter +registry, retained parser state, protocol field or server production change. +One precise guard rejects negative parsed years that wrap beyond a dynamic +zone's recurring-rule cutoff. Current server source otherwise reaches +`ConcurrentIntHashMap` with a negative year key and throws unchecked +`IllegalArgumentException`; schema mode must report a typed invalid value +instead of accepting bytes the legacy conversion cannot produce. This is a +bounded compatibility guard, not a general defensive layer. + +Client and server consume the same 53-case corpus, SHA-256 +`048c9360eb48eeda7785c7a0accc72ef3f87361ea9219fc7c6bddafa92f2f3a9`. +It covers raw long forms and the null sentinel, every grammar, numeric and +named zones, daylight-saving gaps and overlaps, hour 24, greedy fractional +milliseconds, year zero and negative years, wrapping arithmetic, invalid +calendar/time fields, missing or unknown zones, trailing input and Unicode +digits. A deterministic 100,015-input differential matches the current server +through its normal return/error domain. The server's unchecked extreme-year +named-zone crash is separately pinned as a local typed rejection. + +Component tests assert exact DATE wire type, bitmap and raw millisecond values +for the corpus, plus duplicate, omission, rollback, reset, inference and target +metadata errors. Public socket tests prove exact target wire and rollback, and +that a completed DATE block retains its schema snapshot before the next block +rebinds to VARCHAR. Real-server tests write every valid corpus value through +public Sender calls, wait for ACK, drain WAL and compare exact raw DATE values +and nullness; a second A/error/C test proves partial-row cancellation. The +existing successful DATE coercion test and its now-local public rejection test +also pass: four focused server tests total. + +Expanded client acceptance passes 443 tests with zero failures, errors or +skips. Packaged-JAR verification passes the three DATE component tests and both +packaging integration tests. The released-binary matrix passes current client +to QuestDB 10.0.1 in legacy mode, client 1.3.9 to current server in legacy mode, +and current client to current server in schema mode. Current artifact hashes +are client `58836b2fb469c87e264ff2d5f1a7f17f8e6ff9ad5ace3f6f0e5014034b44022d` +and server `dec115136d83b45f4f390bee7b50337fb0da7ee93cd6ae6c0427c0794fe31ce9d`; +the run used `/mnt/pcie5/qwp-string-date-compat/run.OpLD7e`. + +The full `QwpSenderE2ETest` remains diagnostic: its intentional conversion +error count falls from 33 to 32 because both DATE tests are now green. That run +also encountered two independent server `No space left` errors and produced +cascading assertion failures, so its unrelated failure count is not acceptance +evidence. The final intentional conversion implementation left is ranked +DOUBLE-array identity. BINARY parser paths, LONG_ARRAY, non-ASCII CHAR text and +malformed decimal metadata remain explicit server-contract decisions. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 9d935ab69..39233c138 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,10 +1,10 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 66. Scope: QWP v1 over +Status: implemented on the development branch, revision 67. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.42. Iteration -2.43 (INT-to-IPv4) is locally validated. A released-binary +protocol, Sender integration and conversions through iteration 2.43. Iteration +2.44 (STRING-to-DATE) is locally validated. A released-binary compatibility gate is implemented and locally green. The remaining public-setter conversion contract is not complete. @@ -39,9 +39,9 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.43 adds INT input to an IPv4 target. Its -pre-iteration baseline is client `1630c04775` and server `50d487e8b5`, whose -submodule pins that exact client revision. INT-to-IPv4 is the current locally +Current checkpoint: iteration 2.44 adds STRING input to a DATE target. Its +pre-iteration baseline is client `19557e2f` and server `5338ac0845`, whose +submodule pins that exact client revision. STRING-to-DATE is the current locally validated increment. The wider conversion inventory below is not complete; this is not release acceptance. @@ -188,6 +188,21 @@ public-Sender ingestion, partial-row rollback, parameter rejection and a DECIMAL-to-IPv4 schema rebind cover the slice. D064 records the decision and evidence. +STRING input now selects target-native DATE wire representation after local +parsing. It preserves the server's current five-step grammar order: PostgreSQL +date-time, date-only, date with zone, date-time with milliseconds and zone, +UTC date-time with exactly three fractional digits, then raw epoch +milliseconds. `Long.MIN_VALUE`, whether parsed as raw text or produced by +calendar arithmetic, becomes a bitmap NULL. The parser reuses the existing +timestamp calendar and timezone tables with millisecond arithmetic; it does +not add a general format compiler or allocate a date object per value. A narrow +guard rejects an extreme negative-year/named-zone case where the current server +throws an unchecked exception instead of returning a conversion error. A +53-case shared corpus, a 100,015-input differential over the server's normal +return/error domain, exact wire checks, public-Sender ingestion, partial-row +rollback and a DATE-to-VARCHAR generation rebind cover the slice. D065 records +the decision and evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -964,8 +979,8 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.43 on 2026-09-14. The pre-iteration baseline is -client `1630c04775` and server `50d487e8b5`, whose submodule pins that exact +refreshed through iteration 2.44 on 2026-09-14. The pre-iteration baseline is +client `19557e2f` and server `5338ac0845`, whose submodule pins that exact client revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; @@ -1000,7 +1015,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL, decimals, IPv4 | None | | `longColumn` (LONG) | Numeric, text, SYMBOL, DATE, timestamps, decimals | None | | `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | -| `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, timestamps, CHAR, LONG256, geohash, decimals | DATE | +| `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, DATE, timestamps, CHAR, LONG256, geohash, decimals | None | | `symbol` (SYMBOL) | Text, SYMBOL | None | | `timestampColumn(long, unit)` (TIMESTAMP or TIMESTAMP_NANOS) | Timestamps, text | None | | `timestampColumn(Instant)` (currently TIMESTAMP micros in legacy Sender) | Timestamps, preserving nanos for the schema-mode nano target; text | None | @@ -1019,7 +1034,8 @@ DATE also exists as a wire input, but there is no public WebSocket `dateColumn` setter. The server accepts DATE-to-DATE and DATE-to-text only. Neither has a binding entrypoint today; record this wire-only coverage without adding a new public API just to mirror the protocol. DATE **targets** from existing integer -setters are implemented in iteration 2.40; STRING-to-DATE remains missing. +setters are implemented in iteration 2.40; STRING-to-DATE is implemented in +iteration 2.44. The binding still lacks target representation selection for arrays and rejects extension parameters. Adding array append code alone therefore does not @@ -1730,6 +1746,14 @@ They do not change the compatibility contract. INT null and IPv4 zero to the target bitmap, then append every other bit pattern through the existing IPv4 column. Do not add address parsing, widening, an address object or a general numeric-to-IPv4 rule. +26. **Extend the fixed temporal parser, not the abstraction surface.** DATE + needs the server's five fixed parse attempts and millisecond arithmetic. + Reuse the timestamp parser's calendar and timezone metadata, while keeping + DATE's grammar order, raw-millisecond fallback and null sentinel explicit. + Do not add a format compiler, `Instant`/date object allocation, retained + parser state or another conversion registry. Reject the one proven + extreme-year/named-zone server crash precisely; do not turn it into a + general defensive validation layer. These opportunities do not justify a per-setter opt-out, raw-value fallback or send-time transformation. Partial activation is permitted on the unreleased @@ -1745,7 +1769,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.43 +### Plan after iteration 2.44 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1794,12 +1818,18 @@ target width through the established exact integer-decimal tail. The target's declared precision and scale control rescaling and storage; no new decimal representation or server production code was added. -Iteration 2.43 is complete locally: INT selects target-native IPv4 storage, +Iteration 2.43 is committed: INT selects target-native IPv4 storage, normalizing the INT and IPv4 null sentinels without adding parsing, allocation or a general numeric/address conversion. -The next bounded slice is STRING-to-DATE parsing. DOUBLE arrays follow only -after rank metadata is proven end to end. +Iteration 2.44 is complete locally: STRING parses through the server's fixed +DATE grammar sequence and selects target-native DATE storage. Calendar and +timezone mechanics are reused from the existing fixed timestamp parser; no +general date-format framework was added. + +The final intentional conversion slice is DOUBLE-array identity. Implement it +only after proving the rank metadata and every public array representation end +to end; rank and shape validation are part of the conversion contract. BINARY-to-parser conversions, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, not client implementation backlog. From bc16c16439c1348e78e612c016dad5485692e753 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Mon, 14 Sep 2026 19:56:13 +0200 Subject: [PATCH 14/51] feat(qwp): support schema-aware double arrays --- .../cutlass/line/array/AbstractArray.java | 15 +- .../qwp/client/QwpWebSocketSender.java | 20 +- .../qwp/protocol/QwpSchemaBinding.java | 112 +++++- .../cutlass/qwp/protocol/QwpTableBuffer.java | 10 +- .../QwpSchemaSenderIntegrationTest.java | 88 ++++ .../QwpSchemaBindingDoubleArrayTest.java | 375 ++++++++++++++++++ design/schema-aware-sender-journal.md | 91 +++++ design/schema-aware-sender.md | 118 +++--- 8 files changed, 773 insertions(+), 56 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java b/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java index 10c56d5a8..026543481 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java @@ -147,6 +147,19 @@ public void clear() { memA = array.startMemoryA(); } + /** + * Returns the number of dimensions in this array. + * + * @return the array dimensionality + * @throws LineSenderException if the array is closed + */ + public int getDimensionality() { + if (closed) { + throw new LineSenderException("Cannot inspect a closed array"); + } + return array.getDimCount(); + } + /** * Closes this array and releases all associated native memory resources. *

      @@ -326,4 +339,4 @@ protected int toFlatOffset(int[] coords) { } return flatOffset; } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 7c21812b0..ad68068a3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1762,13 +1762,16 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + binding.doubleArray(name, values); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1784,13 +1787,16 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + binding.doubleArray(name, values); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1806,13 +1812,16 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + binding.doubleArray(name, values); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; @@ -1828,13 +1837,16 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { try { QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { - binding.unsupportedColumn(name, "DOUBLE_ARRAY"); + binding.doubleArray(name, array); return this; } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(array); } + } catch (LineSenderSchemaException e) { + rollbackRow(); + throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { rollbackRow(); throw e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index da495a2af..a12f3c11c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -28,6 +28,7 @@ import io.questdb.client.cairo.MicrosTimestampDriver; import io.questdb.client.cairo.NanosTimestampDriver; import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.std.Chars; import io.questdb.client.std.Decimal; import io.questdb.client.std.Decimal128; @@ -158,7 +159,7 @@ public QwpSchemaBinding decimalColumn(CharSequence name, CharSequence value, Dec int index = targetIndex(name, "DECIMAL256"); int targetType = targetType(index, ColumnType.DECIMAL256); if (index < 0) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(name); + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(name); if (inferred != null && inferred.getSize() > buffer.getRowCount()) { return this; } @@ -264,6 +265,59 @@ public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { return this; } + public QwpSchemaBinding doubleArray(CharSequence name, double[] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 1); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[][] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 2); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[][][] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 3); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, DoubleArray array) { + buffer.requireSchemaBinding(this); + if (array == null) { + return this; + } + int index = targetIndex(name, "DOUBLE_ARRAY"); + QwpTableBuffer.ColumnBuffer existing = buffer.getExistingColumnByName(name); + if (existing != null && existing.getSize() > buffer.getRowCount()) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, index, array.getDimensionality()); + if (column != null) { + column.addDoubleArray(array); + } + return this; + } + public QwpSchemaBinding byteColumn(CharSequence name, byte value) { return integerColumn(name, value, "BYTE", ColumnType.BYTE, false); } @@ -320,7 +374,7 @@ public QwpSchemaBinding designatedTimestamp(Instant value) { private QwpTableBuffer.ColumnBuffer designatedColumn(int targetType, String inputType) { byte wireType = timestampWireType(targetType); if (schema.getResult() == QwpSchemaProtocol.RESULT_MISSING) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(""); + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(""); if (inferred != null && inferred.getType() != wireType) { throw unsupported(null, inputType, -1, "inferred designated timestamp type conflict [inferredType=" + QwpConstants.getTypeName(inferred.getType()) + ']'); @@ -787,7 +841,7 @@ public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { public QwpSchemaBinding unsupportedColumn(CharSequence name, String inputType) { buffer.requireSchemaBinding(this); int index = targetIndex(name, inputType); - QwpTableBuffer.ColumnBuffer inferred = index < 0 ? buffer.getExistingInferredColumn(name) : null; + QwpTableBuffer.ColumnBuffer inferred = index < 0 ? buffer.getExistingColumnByName(name) : null; if (inferred != null && inferred.getSize() > buffer.getRowCount()) { return this; } @@ -900,6 +954,46 @@ private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String input return column; } + private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn(CharSequence name, int sourceDimensions) { + int index = targetIndex(name, "DOUBLE_ARRAY"); + return targetDoubleArrayColumn(name, index, sourceDimensions); + } + + private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn( + CharSequence name, + int index, + int sourceDimensions + ) { + int targetType = targetType(index, ColumnType.encodeArrayType(ColumnType.DOUBLE, sourceDimensions)); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "DOUBLE_ARRAY", index, targetType); + if (column == null) { + return null; + } + if (!ColumnType.isArray(targetType) + || ColumnType.decodeArrayElementType(targetType) != ColumnType.DOUBLE) { + throw unsupported(name, "DOUBLE_ARRAY", targetType, "conversion is not implemented"); + } + int targetDimensions = ColumnType.decodeWeakArrayDimensionality(targetType); + if (targetDimensions < 1 + || targetType != ColumnType.encodeArrayType(ColumnType.DOUBLE, targetDimensions)) { + throw unsupported(name, "DOUBLE_ARRAY", targetType, "invalid target array type"); + } + if (targetDimensions != sourceDimensions) { + throw unsupported(name, "DOUBLE_ARRAY", targetType, + "array dimensionality mismatch [sourceDims=" + sourceDimensions + + ", targetDims=" + targetDimensions + ']'); + } + if (index < 0) { + int inferredDimensions = column.pinInferredArrayDimensionality(sourceDimensions); + if (inferredDimensions != sourceDimensions) { + throw unsupported(name, "DOUBLE_ARRAY", targetType, + "array dimensionality mismatch [sourceDims=" + sourceDimensions + + ", inferredDims=" + inferredDimensions + ']'); + } + } + return column; + } + private QwpTableBuffer.ColumnBuffer targetColumn( CharSequence name, String inputType, @@ -907,7 +1001,7 @@ private QwpTableBuffer.ColumnBuffer targetColumn( int targetType ) { if (index < 0) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(name); + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(name); if (inferred != null && inferred.getSize() > buffer.getRowCount()) { return null; } @@ -934,6 +1028,14 @@ private QwpTableBuffer.ColumnBuffer targetColumn( } private byte wireType(int targetType) { + if (ColumnType.isArray(targetType)) { + int dimensions = ColumnType.decodeWeakArrayDimensionality(targetType); + return dimensions >= 1 + && ColumnType.decodeArrayElementType(targetType) == ColumnType.DOUBLE + && targetType == ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions) + ? QwpConstants.TYPE_DOUBLE_ARRAY + : 0; + } if (ColumnType.isGeoHash(targetType)) { int bits = ColumnType.getGeoHashBits(targetType); return bits >= 1 && bits <= 60 && targetType == ColumnType.getGeoHashTypeWithBits(bits) @@ -1027,7 +1129,7 @@ private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String } int index = targetIndex(name, inputType); if (index < 0) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingInferredColumn(name); + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(name); if (inferred != null && inferred.getSize() > buffer.getRowCount()) { return this; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 032bef822..398a02136 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -193,7 +193,7 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { return lookupColumn(name, type); } - ColumnBuffer getExistingInferredColumn(CharSequence name) { + ColumnBuffer getExistingColumnByName(CharSequence name) { int index = columnNameToIndex.get(name); return index >= 0 ? columns.get(index) : null; } @@ -664,6 +664,7 @@ public static class ColumnBuffer implements QuietCloseable { // Decimal storage private byte decimalScale = -1; private boolean schemaDecimalScaleLocked; + private int inferredArrayDimensionality = -1; private double[] doubleArrayData; // GeoHash precision (number of bits, 1-60) private int geohashPrecision = -1; @@ -1534,6 +1535,13 @@ public int getSize() { return size; } + int pinInferredArrayDimensionality(int dimensionality) { + if (inferredArrayDimensionality < 0) { + inferredArrayDimensionality = dimensionality; + } + return inferredArrayDimensionality; + } + public long getStringDataAddress() { return stringData != null ? stringData.pageAddress() : 0; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index e8e1d93ce..12cd52ff7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -920,6 +920,58 @@ public void testDateGenerationRemainsPinnedBeforeVarcharRebind() throws Exceptio }); } + @Test + public void testDoubleArrayUsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1181, 1191, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.doubleArray("value", new double[]{99.0}); + Assert.fail("expected array rank mismatch"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events", 1181, 1191); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleArrayGenerationRemainsPinnedAcrossRankChange() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1201, 1211, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + handler.targetType = array1d; + handler.version = 1212; + sender.table("events"); + try { + sender.doubleArray("value", new double[]{3.0, 4.0}); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.doubleArray("value", new double[]{3.0, 4.0}).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .twoDoubleArrayRankBlocks("events", 1201, 1211, 1212); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + @Test public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { assertMemoryLeak(() -> { @@ -2904,6 +2956,42 @@ private void stringDateTable( eof(); } + private void doubleArrayTable(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{1, 2}, 1.0, 2.0); + assertDoubleArray(new int[]{2, 1}, 3.0, 4.0); + eof(); + } + + private void twoDoubleArrayRankBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{1, 2}, 1.0, 2.0); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{2}, 3.0, 4.0); + eof(); + } + + private void assertDoubleArray(int[] shape, double... values) { + Assert.assertEquals(shape.length, u8()); + for (int dimension : shape) { + Assert.assertEquals(dimension, in.getInt()); + } + for (double value : values) { + Assert.assertEquals(Double.doubleToRawLongBits(value), in.getLong()); + } + } + private void twoDateAndVarcharBlocks( String table, int tableId, long firstVersion, long secondVersion ) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java new file mode 100644 index 000000000..a889eb4fc --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java @@ -0,0 +1,375 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingDoubleArrayTest { + + @Test + public void testAllPublicRepresentationsUseExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + int[] shape32 = new int[ColumnType.ARRAY_NDIMS_LIMIT]; + java.util.Arrays.fill(shape32, 1); + double payloadNaN = Double.longBitsToDouble(0x7ff8000000000042L); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray array = new DoubleArray(shape32)) { + array.append(10.5); + QwpSchemaBinding binding = binding(buffer, + column("a1", arrayType(1)), + column("a2", arrayType(2)), + column("a3", arrayType(3)), + column("a32", arrayType(ColumnType.ARRAY_NDIMS_LIMIT))); + binding.doubleArray("a1", new double[]{payloadNaN, -0.0, Double.POSITIVE_INFINITY}); + binding.doubleArray("a2", new double[][]{{3.0, 4.0}, {5.0, 6.0}}); + binding.doubleArray("a3", new double[][][]{{{7.0}, {8.0}}}); + binding.doubleArray("a32", array); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 1, 4); + columnDef(reader, "a1"); + columnDef(reader, "a2"); + columnDef(reader, "a3"); + columnDef(reader, "a32"); + array(reader, new int[]{3}, payloadNaN, -0.0, Double.POSITIVE_INFINITY); + array(reader, new int[]{2, 2}, 3.0, 4.0, 5.0, 6.0); + array(reader, new int[]{1, 2, 1}, 7.0, 8.0); + array(reader, shape32, 10.5); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNullRepresentationsAreNoopsAndWrapperRankTracksReshape() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray array = new DoubleArray(1, 1)) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.doubleArray("a", (double[]) null) + .doubleArray("b", (double[][]) null) + .doubleArray("c", (double[][][]) null) + .doubleArray("d", (DoubleArray) null); + Assert.assertEquals(0, buffer.getColumnDefs().length); + + Assert.assertEquals(2, array.getDimensionality()); + array.reshape(1); + Assert.assertEquals(1, array.getDimensionality()); + array.append(42.0); + binding.doubleArray("value", array); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 1, 1, -1, -1); + columnDef(reader, "value"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 1, new int[]{1}, new double[]{42.0}); + Assert.assertEquals(size, reader.position()); + } + }); + + try (DoubleArray array = new DoubleArray(1)) { + array.close(); + Assert.assertThrows(LineSenderException.class, array::getDimensionality); + } + } + + @Test + public void testKnownRankDuplicateRollbackAndShapeValidation() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray closedDuplicate = new DoubleArray(1)) { + QwpSchemaBinding binding = binding(buffer, + column("v", arrayType(1)), column("m", arrayType(2))); + + binding.doubleArray("v", new double[]{1.0}); + binding.doubleArray("v", new double[][]{{99.0}}); // duplicate: first value wins + closedDuplicate.close(); + binding.doubleArray("v", closedDuplicate); // duplicate: do not inspect an ignored value + buffer.nextRow(); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[][]{{2.0}}), + "sourceDims=2", "targetDims=1"); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + Assert.assertThrows(LineSenderException.class, + () -> binding.doubleArray("m", new double[][]{{2.0}, {3.0, 4.0}})); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.doubleArray("v", new double[0]); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 2, 1); + columnDef(reader, "v"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 1, new int[]{1}, new double[]{1.0}); + rawArray(reader, 1, new int[]{0}, new double[0]); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testMissingSchemaPinsFirstEffectiveRank() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.doubleArray("v", new double[][]{{1.0, 2.0}}); + buffer.nextRow(); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{3.0}), + "sourceDims=1", "inferredDims=2"); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.doubleArray("v", new double[][]{{4.0}, {5.0}}); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 2, 1, -1, -1); + columnDef(reader, "v"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 2, new int[]{1, 2}, new double[]{1.0, 2.0}); + rawArray(reader, 2, new int[]{2, 1}, new double[]{4.0, 5.0}); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{6.0}), + "sourceDims=1", "inferredDims=2"); + } + }); + } + + @Test + public void testRejectsWrongElementTypeWeakRankParametersAndDesignatedColumn() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.encodeArrayType(ColumnType.LONG, 1))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "targetType=LONG[]"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + int weak = arrayType(1) | (1 << 19); + QwpSchemaBinding binding = binding(buffer, column("v", weak)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "conversion is not implemented"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("v", arrayType(1), new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "parameterized target type"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("v", arrayType(1))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "designated timestamp writes"); + } + } + + private static void array(Reader reader, int[] shape, double... values) { + Assert.assertEquals(0, reader.u8()); + rawArray(reader, shape.length, shape, values); + } + + private static int arrayType(int dimensions) { + return ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return binding(buffer, -1, columns); + } + + private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designated, columns)); + } + + private static byte[] column(String name, int type) { + return column(name, type, new byte[0]); + } + + private static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 6 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type) + .putShort((short) params.length).put(params).array(); + } + + private static void columnDef(Reader reader, String name) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE_ARRAY, reader.u8()); + } + + private static QwpSchemaResponse decode(byte[] payload) { + ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload); + long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.capacity(); i++) { + Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); + } finally { + Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); + } + } + + private static QwpSchemaResponse known(int designated, byte[]... columns) { + int length = 26; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + private static QwpSchemaResponse missing() { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); + } + + private static void rawArray(Reader reader, int expectedDims, int[] shape, double[] values) { + Assert.assertEquals(expectedDims, reader.u8()); + for (int dimension : shape) { + Assert.assertEquals(dimension, reader.i32()); + } + for (double value : values) { + Assert.assertEquals(Double.doubleToRawLongBits(value), reader.i64()); + } + } + + private static void assertReason( + LineSenderSchemaException.Reason reason, + Runnable action, + String... messageParts + ) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(reason, error.getReason()); + for (String part : messageParts) { + Assert.assertTrue(error.getMessage(), error.getMessage().contains(part)); + } + } + + private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + return tableHeader(encoder, size, rows, columns, 1, 1); + } + + private static Reader tableHeader( + QwpWebSocketEncoder encoder, + int size, + int rows, + int columns, + int tableId, + long version + ) { + Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + if (tableId < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(tableId, reader.i32()); + Assert.assertEquals(version, reader.i64()); + } + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + return reader; + } + + private static final class Reader { + private final long address; + private final int limit; + private int position; + + private Reader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + private int i32() { + Assert.assertTrue(position <= limit - Integer.BYTES); + int value = Unsafe.getUnsafe().getInt(address + position); + position += Integer.BYTES; + return value; + } + + private long i64() { + Assert.assertTrue(position <= limit - Long.BYTES); + long value = Unsafe.getUnsafe().getLong(address + position); + position += Long.BYTES; + return value; + } + + private int position() { + return position; + } + + private void skip(int length) { + position += length; + Assert.assertTrue(position <= limit); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = (byte) u8(); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private int u8() { + Assert.assertTrue(position < limit); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + for (; shift < 32; shift += 7) { + int b = u8(); + result |= (b & 0x7f) << shift; + if ((b & 0x80) == 0) { + return result; + } + } + throw new AssertionError("unterminated varint"); + } + } +} diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 90f8eecf7..621ee509e 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7718,3 +7718,94 @@ cascading assertion failures, so its unrelated failure count is not acceptance evidence. The final intentional conversion implementation left is ranked DOUBLE-array identity. BINARY parser paths, LONG_ARRAY, non-ASCII CHAR text and malformed decimal metadata remain explicit server-contract decisions. + +## D066 — DOUBLE arrays reuse the existing wire path and pin rank + +Status: iteration 2.45 implemented and locally validated, 2026-09-14. Design +revision 68. The pre-iteration baseline is client `47a37a6f` and server +`c1f970220f`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +All four public `doubleArray` representations now participate in schema mode. +The primitive overloads have ranks 1, 2 and 3. The N-dimensional `DoubleArray` +wrapper exposes its current rank through an O(1) accessor, including after +`reshape`. A known target must be a canonical strong DOUBLE array of exactly +that rank. A confirmed missing column infers DOUBLE_ARRAY and pins its first +effective rank in the existing column buffer across resets; a later different +rank fails locally rather than creating a mixed-rank block. Null references +remain no-ops, empty shapes remain values, and the existing duplicate-first, +row rollback and one-refresh rules are preserved. Duplicate suppression runs +before inspecting a wrapper's rank, so an invalid or closed ignored second +value cannot change first-value-wins behavior. + +This is rank-checked identity, not an element conversion. The binding selects +the existing `TYPE_DOUBLE_ARRAY` column, and the established buffer and encoder +continue to own shape validation, element copying and wire output. No converter, +second shape model, per-row object, array extension parameter, send-time +transformation or server production change was added. The wrapper accessor +avoids a second traversal or temporary capture before validation. The only new +retained state is one integer on a confirmed-missing inferred column; known +schemas already carry rank in the encoded `ColumnType`. + +No throughput claim is made. There was no successful schema-aware array path +to compare before this slice, while the legacy append and encode loops are +unchanged. The new successful path adds only constant-time schema/rank checks +before the same element copy, so a synthetic A/B number would not isolate a +real regression boundary. + +The exclusion of LONG arrays is source-backed. The current server validates +DOUBLE_ARRAY batch rank but does not compare a cursor's element type with an +existing array target. `WalColumnarRowAppender.putArrayColumn` then constructs +storage from the cursor element type. Auto-creation rejects LONG_ARRAY, and SQL +does not currently create LONG[] columns. Accepting DOUBLE_ARRAY-to-LONG[] or +enabling `longArray` in schema mode would therefore reproduce a server +validation gap, not implement a supported cast. Synthetic schema tests pin the +local rejection because a real-server LONG[] fixture is not constructible. + +Five component tests cover every public representation, ranks 1, 2, 3 and 32, +live wrapper reshaping, all null overloads, empty shapes, raw NaN payload bits, +signed zero, infinity, duplicates, ragged inputs, rollback, reset, missing-rank +pinning, wrong element type, weak rank, parameters and designated-column +rejection. Three focused public-socket tests cover exact target wire bytes, +A/error/C rollback, the standard one-refresh path, generation pinning and null +no-op before negotiation. The expanded client acceptance selection passes 494 +tests with zero failures, errors or skips. Packaged-JAR verification passes the +five component tests and both packaging integration tests. + +The focused real-server gate passes nine tests: two new tests cover ranks 1 +through 4, all public representations, empty shapes, exact stored shapes, ACKs +and partial-row recovery; seven existing array/rejection tests now assert the +new early local errors. The complete 137-test `QwpSenderE2ETest` diagnostic has +113 passing tests, two assertion failures and 22 early schema errors. None is +an unimplemented intentional conversion: 17 are old aggregate rejection tests, +three are specialized tests that still expect a server-side error stage, two +are the explicit CHAR-to-text boundary, and the assertion failures are stale +empty/null-column-name message checks. With the test JVM started on +`/mnt/pcie5`, this run has no disk-space infrastructure failures. + +The released-binary matrix passes current client to QuestDB 10.0.1 in legacy +mode, client 1.3.9 to current server in legacy mode, and current client to +current server in schema mode. Current artifact hashes are client +`6d3cb13f4058e1cf9a2af7b3459266a2beffb4a92e7f148880dc5fcf40b24292` +and server +`87df2cee84f1cad1c450831fc2ec6f995437cb5eda701486bf452a8c3a3da524`; +the run used `/mnt/pcie5/qwp-double-array-compat-final/run.pK2tjd`. + +No array-specific store-and-forward test was added. Schema mode writes the +same self-contained DOUBLE_ARRAY bytes before persistence, and the 494-test +gate includes schema network replay, self-sufficient-frame and recovery replay +coverage. Repeating that unchanged persistence path with another value family +would not add a new contract assertion. + +A full client-module diagnostic encountered the existing +`QuestDBServerRecoveryTest.testFacadeStartsWhileServerDownThenWritesAndReaderConnectsOnRecovery` +expectation: the first effective write now waits for mandatory schema +negotiation and times out while no server exists. The timed-out test leaves its +reconnecting sender active, so that diagnostic run was stopped. This is a +separate startup-contract fixture, not an array regression; the 494-test +acceptance set is green. + +The intentional, server-supported public-setter conversion catalogue is now +complete. BINARY parser paths, LONG_ARRAY, non-ASCII CHAR text and malformed +decimal metadata remain explicit server-contract decisions, not unfinished +client converters. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 39233c138..d33ec6d37 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,12 +1,13 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 67. Scope: QWP v1 over +Status: implemented on the development branch, revision 68. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.43. Iteration -2.44 (STRING-to-DATE) is locally validated. A released-binary -compatibility gate is implemented and locally green. The remaining public-setter -conversion contract is not complete. +protocol, Sender integration and conversions through iteration 2.44. Iteration +2.45 (ranked DOUBLE-array identity) is locally validated. A released-binary +compatibility gate is implemented and locally green. The intentional, +server-supported public-setter conversion catalogue is complete. The explicit +server-contract boundaries below remain excluded rather than emulated. ## Goal and contract @@ -39,11 +40,14 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.44 adds STRING input to a DATE target. Its -pre-iteration baseline is client `19557e2f` and server `5338ac0845`, whose -submodule pins that exact client revision. STRING-to-DATE is the current locally -validated increment. -The wider conversion inventory below is not complete; this is not release acceptance. +Current checkpoint: iteration 2.45 adds all public DOUBLE-array representations +to canonical DOUBLE-array targets of exactly the same rank. Its pre-iteration +baseline is client `47a37a6f` and server `c1f970220f`, whose submodule pins that +exact client revision. The client reuses the existing array buffer and encoder, +validates known rank before append, and pins the first effective rank for a +confirmed missing column. The intentional conversion inventory below is now +complete. This does not turn the separately documented unsafe or undefined +server paths into supported conversions. The standing compatibility gate now runs three real process combinations: current client against QuestDB 10.0.1, client 1.3.9 against the current server, @@ -203,6 +207,18 @@ return/error domain, exact wire checks, public-Sender ingestion, partial-row rollback and a DATE-to-VARCHAR generation rebind cover the slice. D065 records the decision and evidence. +All public DOUBLE-array inputs now select target-native DOUBLE_ARRAY storage +when the server schema carries a canonical strong DOUBLE array of the same rank. +The primitive overloads supply ranks 1, 2 and 3; `DoubleArray` exposes its live +rank for dimensions 1 through 32. Known-rank mismatches fail before append, and +a confirmed missing column pins the first effective rank in its existing buffer +definition so later rows cannot form a mixed-rank block. Null references remain +no-ops, empty shapes remain values, and duplicate first-value-wins behavior is +unchanged. The implementation reuses the existing shape/data buffer and encoder; +it adds no element converter, copy, array parameter format or server production +code. Exact-wire, public socket, real-server shape/rank and partial-row recovery +tests cover the slice. D066 records the decision and evidence. + Iteration 2.15 is accepted as a bounded, unreleased Sender integration. Public setters select the negotiated mode, use server-directed conversions, cancel failed partial rows and retain completed rows under their original @@ -979,8 +995,8 @@ when the table or column is absent. ## Conversion backlog (server-source audit) Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.44 on 2026-09-14. The pre-iteration baseline is -client `19557e2f` and server `5338ac0845`, whose submodule pins that exact +refreshed through iteration 2.45 on 2026-09-14. The pre-iteration baseline is +client `47a37a6f` and server `c1f970220f`, whose submodule pins that exact client revision. The original audit bases were client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server `12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; @@ -1027,7 +1043,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | | `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | All decimals, text | None; this is not the `stringColumn` parser path | | `geoHashColumn(long, bits)` and `geoHashColumn(CharSequence)` (GEOHASH with source bits) | Geohash with exactly matching bit precision, text | None | -| `doubleArray` (all Java ranks and `DoubleArray`; DOUBLE_ARRAY) | None | DOUBLE array of the target rank; identity/shape validation, not element casting | +| `doubleArray` (all Java ranks and `DoubleArray`; DOUBLE_ARRAY) | DOUBLE array of exactly matching rank | None | | `longArray` (all Java ranks and `LongArray`; LONG_ARRAY) | None | No supported conversion contract established; see the existing-server gap below | DATE also exists as a wire input, but there is no public WebSocket `dateColumn` @@ -1035,17 +1051,12 @@ setter. The server accepts DATE-to-DATE and DATE-to-text only. Neither has a binding entrypoint today; record this wire-only coverage without adding a new public API just to mirror the protocol. DATE **targets** from existing integer setters are implemented in iteration 2.40; STRING-to-DATE is implemented in -iteration 2.44. - -The binding still lacks target representation selection for arrays and rejects -extension parameters. Adding array append code alone therefore does not -complete those pairs: select the target wire representation and pin its rank -before append. Preserve the existing non-owning binding and buffer ownership. +iteration 2.44. Ranked DOUBLE-array identity is implemented in iteration 2.45. -### Rules the missing conversions must cover +### Conversion rules -These are server-source requirements to test, not permission to silently add -new casts or copy the unresolved behaviours below. +These are server-source requirements, not permission to silently add new casts +or copy the unresolved behaviours below. - **Integer sources:** retain distinct BYTE/SHORT/INT/LONG input contracts. Numeric narrowing checks range. DATE takes raw epoch milliseconds; ordinary @@ -1143,16 +1154,18 @@ new casts or copy the unresolved behaviours below. first produces a precision-bearing GEOHASH and must not be confused with `stringColumn`. Geohash-to-text emits bit strings for valid positive wire precision, not the helper's unreachable negative-precision character branch. -- **Arrays:** DOUBLE_ARRAY copies 8-byte elements; rank must match the known - target, with per-row lengths permitted to vary. The cursor validates rank - 1..32, nonnegative dimensions, checked element count and exact payload size. - Test each Java overload and wrapper, omitted/null/empty arrays, mixed ranks, - ragged inputs and capacity failures. No implicit LONG/DOUBLE element cast or - rank conversion is established by the server. - -Every new pair also needs local rejection/rollback, exact target-wire and -stored-value checks, recovery coverage, and explicit legacy comparisons using -shared vectors. In particular, the fixed-width server cursor treats a null +- **Arrays:** DOUBLE_ARRAY copies raw 8-byte elements; the canonical encoded + target type supplies both DOUBLE element type and strong rank. Rank must match + exactly, while per-row lengths may vary. A confirmed missing column pins the + first effective rank across rows and batches. Public null array references are + no-ops; omission produces SQL NULL and an empty shape is a value. The cursor + validates rank 1..32, nonnegative dimensions, checked element count and exact + payload size. No implicit LONG/DOUBLE element cast or rank conversion is + established by the server. + +Every implemented pair needs local rejection/rollback, exact target-wire and +stored-value checks, relevant recovery coverage, and explicit legacy comparisons +using shared vectors. In particular, the fixed-width server cursor treats a null bitmap as authoritative and otherwise recognizes type-specific sentinels. BYTE/SHORT/CHAR have no source sentinel; INT minimum, LONG minimum, floating NaN, IPv4 zero and UUID/LONG256 sentinel limbs can behave differently when a @@ -1754,13 +1767,21 @@ They do not change the compatibility contract. parser state or another conversion registry. Reject the one proven extreme-year/named-zone server crash precisely; do not turn it into a general defensive validation layer. +27. **Treat DOUBLE-array support as rank validation, not conversion.** Read the + rank directly from the public wrapper, require the canonical strong DOUBLE + array type, and reuse the existing array buffer and encoder. For a missing + column, keep one rank integer on its inferred column definition so later + rows cannot form a mixed-rank block. Do not copy elements into a converter, + add array parameters, build a second shape model, or generalize this into + LONG-array casting. The current server checks rank but can build storage + from the cursor's element type, so accepting a non-DOUBLE target would + reproduce a server validation gap rather than a supported conversion. These opportunities do not justify a per-setter opt-out, raw-value fallback or -send-time transformation. Partial activation is permitted on the unreleased -development branch to exercise the real public Sender path early. It is not a -release-ready client: missing conversion families still reject some -previously working writes. Preserve legacy-mode behavior and complete the -schema-mode contract before release. +send-time transformation. The intentional conversion catalogue is complete on +the unreleased development branch. Release review must still verify the standing +gates and explicitly accept or resolve the server-contract boundaries above; +they must not be hidden behind fallback behavior. ## Iterative implementation and testing @@ -1769,7 +1790,7 @@ implementation and its tests together. Do not wait for negotiation, compatibility and all converters to be implemented before exercising the complete client-to-server path. Reuse the existing test infrastructure. -### Plan after iteration 2.44 +### Plan after iteration 2.45 The permanent compatibility gate is implemented. It launches released and current artifacts as separate processes and checks these observable contracts: @@ -1822,17 +1843,24 @@ Iteration 2.43 is committed: INT selects target-native IPv4 storage, normalizing the INT and IPv4 null sentinels without adding parsing, allocation or a general numeric/address conversion. -Iteration 2.44 is complete locally: STRING parses through the server's fixed +Iteration 2.44 is committed: STRING parses through the server's fixed DATE grammar sequence and selects target-native DATE storage. Calendar and timezone mechanics are reused from the existing fixed timestamp parser; no general date-format framework was added. -The final intentional conversion slice is DOUBLE-array identity. Implement it -only after proving the rank metadata and every public array representation end -to end; rank and shape validation are part of the conversion contract. -BINARY-to-parser conversions, LONG_ARRAY, non-ASCII -CHAR-to-VARCHAR and malformed decimal metadata remain server-contract decisions, -not client implementation backlog. +Iteration 2.45 is complete locally: every public DOUBLE-array overload selects +the existing DOUBLE_ARRAY wire representation only when the server's canonical +element type and rank match. The N-dimensional wrapper exposes its current rank; +confirmed missing columns pin their first effective rank. Exact-wire tests cover +ranks 1, 2, 3 and 32, raw floating-point bits, empty shapes, duplicate/rollback +behavior and generation changes. Real-server tests cover public ranks 1 through +4, stored shapes, rank rejection and partial-row recovery. + +There is no remaining intentional client conversion slice. Further datatype +work must start by resolving one of the explicit server-contract boundaries: +BINARY-to-parser behavior, LONG_ARRAY validation, non-ASCII CHAR text output or +malformed decimal metadata. Until then, rejecting those paths is simpler and +safer than copying an accidental server behavior into the client. Every slice keeps the existing observable-contract rules below: public API, exact wire, SQL result, partial-row rollback and relevant recovery behavior. From 59ade9169ee004d7cebd19cd74f08f1fc937cab7 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 15 Sep 2026 10:34:25 +0200 Subject: [PATCH 15/51] fix(qwp): preserve async schema startup availability --- .../main/java/io/questdb/client/Sender.java | 10 +- .../qwp/client/QwpWebSocketSender.java | 13 +- .../QwpSchemaSenderIntegrationTest.java | 243 ++++++++++++------ design/schema-aware-sender-journal.md | 129 ++++++++++ design/schema-aware-sender.md | 210 +++++++++++---- 5 files changed, 480 insertions(+), 125 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index eaeb4d005..f49a3303c 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -819,7 +819,15 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Transport failures (unreachable or + * engine until the wire is up. Before schema support is confirmed, + * new rows use the legacy conversion contract; schema mismatches may + * therefore be reported later by the server, and values such as + * nanosecond timestamps use their legacy representation. If recovered + * data already requires schema support, writes instead wait for schema + * negotiation. Once support is confirmed, all future rows use strict + * schema lookup and validation, including across reconnects; a row + * already in progress keeps the contract under which it began. Transport + * failures (unreachable or * dropped server) retry indefinitely and are never surfaced -- the * buffered rows are safe in SF and the server may still appear. A * terminal auth, upgrade or capability rejection on the initial diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index ad68068a3..3c9e46f01 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4503,8 +4503,10 @@ private void ensureConnected() { /** * Selects the wire contract before the first effective value of a row. - * Empty table selection remains local; an unreachable or failed handshake - * is never guessed to be a legacy server. + * Empty table selection remains local. In asynchronous initial-connect mode, + * fresh senders use the legacy contract until schema support is confirmed; + * recovered schema-framed data and senders that have already confirmed support + * retain the strict schema path. */ private QwpSchemaBinding bindingForEffectiveWrite() { QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); @@ -4512,8 +4514,13 @@ private QwpSchemaBinding bindingForEffectiveWrite() { schemaResolutionFresh[0] = false; return pinned; } - final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; ensureConnected(); + if (initialConnectMode == Sender.InitialConnectMode.ASYNC + && !cursorEngine.requiresSchema()) { + schemaResolutionFresh[0] = false; + return null; + } + final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(deadlineNanos))) { return null; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 12cd52ff7..1f60e5e72 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -18,9 +18,12 @@ import io.questdb.client.std.Decimal64; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -32,12 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -47,6 +48,8 @@ public class QwpSchemaSenderIntegrationTest { private static final String DECIMAL_TEXT_CORPUS = "/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv"; + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); @Test public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exception { @@ -1804,45 +1807,40 @@ public void testPendingLegacyRowSurvivesUpgradeAndPublishesBeforeSchemaRow() thr }); } - @Test - public void testAsyncFirstEffectiveWriteWaitsForNegotiationInsteadOfGuessingLegacy() throws Exception { + @Test(timeout = 10_000) + public void testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect() throws Exception { assertMemoryLeak(() -> { int port = TestPorts.findUnusedPort(); Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" - + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;"); - ExecutorService executor = Executors.newSingleThreadExecutor(); - CountDownLatch entered = new CountDownLatch(1); - Future write = executor.submit(() -> { - entered.countDown(); - sender.table("events").uuidColumn("id", 7, 8).atNow(); - sender.flush(); - }); + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); TestWebSocketServer server = null; try { - Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); - try { - write.get(100, TimeUnit.MILLISECONDS); - Assert.fail("first effective write guessed a wire mode without a server handshake"); - } catch (TimeoutException expected) { - // The write remains blocked inside the bounded negotiation path. - } - SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 71, 72); + Instant value = Instant.ofEpochSecond(3, 456_789_123); + // No server is listening: this effective write must nevertheless + // complete on the caller thread using the existing legacy conversion. + sender.table("events").timestampColumn("ts", value).atNow(); + + SchemaHandler handler = new SchemaHandler( + QwpSchemaProtocol.RESULT_KNOWN, 71, 72, true, -1); server = new TestWebSocketServer(handler, false, null, port); server.setAdvertiseSchema(true); server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - try { - write.get(5, TimeUnit.SECONDS); - } catch (ExecutionException e) { - throw new AssertionError(e.getCause()); - } - byte[] frame = handler.awaitDataFrame(); - Assert.assertTrue((frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyTimestampTable( + "events", "ts", 3_456_789L); + new FrameReader(frames.get(1)).schemaTimestampTable( + "events", 71, 72, "ts", 3_456_789_123L); Assert.assertEquals(1, handler.describeRequests.get()); } finally { - executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); sender.close(); if (server != null) { server.close(); @@ -1892,12 +1890,15 @@ public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exceptio } @Test(timeout = 10_000) - public void testFirstNegotiationDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { + public void testSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { assertMemoryLeak(() -> { - int port = TestPorts.findUnusedPort(); - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port - + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" - + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + handler.respondToDescribe = false; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(1_000); sender.table("events"); try { @@ -1906,53 +1907,88 @@ public void testFirstNegotiationDeadlineFailsTypedThenRetryUsesAvailableSchema() } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); } - SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); - try (TestWebSocketServer server = new TestWebSocketServer(handler, false, null, port)) { - server.setAdvertiseSchema(true); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - sender.uuidColumn("id", 3, 4).atNow(); - sender.flush(); - Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] - & QwpConstants.FLAG_SCHEMA) != 0); - } + handler.respondToDescribe = true; + sender.uuidColumn("id", 3, 4).atNow(); + sender.flush(); + Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); } }); } @Test - public void testInterruptedFirstNegotiationFailsTypedAndPreservesInterrupt() throws Exception { + public void testInterruptedSchemaLookupFailsTypedAndPreservesInterrupt() throws Exception { assertMemoryLeak(() -> { - int port = TestPorts.findUnusedPort(); - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port - + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" - + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;"); - ExecutorService executor = Executors.newSingleThreadExecutor(); - CountDownLatch entered = new CountDownLatch(1); - AtomicReference writer = new AtomicReference<>(); - AtomicBoolean interruptPreserved = new AtomicBoolean(); - Future result = executor.submit(() -> { - writer.set(Thread.currentThread()); - entered.countDown(); + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 93, 94); + handler.respondToDescribe = false; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + AtomicReference writer = new AtomicReference<>(); + AtomicBoolean interruptPreserved = new AtomicBoolean(); + Future result = executor.submit(() -> { + writer.set(Thread.currentThread()); + entered.countDown(); + try { + sender.table("events").uuidColumn("id", 1, 2); + return null; + } catch (LineSenderSchemaException e) { + interruptPreserved.set(Thread.currentThread().isInterrupted()); + return e; + } + }); + try { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + writer.get().interrupt(); + LineSenderSchemaException error = result.get(5, TimeUnit.SECONDS); + Assert.assertNotNull(error); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, error.getReason()); + Assert.assertTrue("writer interrupt status was cleared", interruptPreserved.get()); + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAsyncRecoveredSchemaBacklogDoesNotFallBackToLegacy() throws Exception { + assertMemoryLeak(() -> { + File sfRoot = temporaryFolder.newFolder("schema-required-recovery"); + SchemaHandler handler = new SchemaHandler( + QwpSchemaProtocol.RESULT_KNOWN, 95, 96, false); + int unavailablePort; + try (TestWebSocketServer server = schemaServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;close_flush_timeout_millis=0;"; + try (Sender sender = Sender.fromConfig(cfg)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + unavailablePort = TestPorts.findUnusedPort(); + } + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + unavailablePort + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;initial_connect_retry=async;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;")) { + ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(250); + sender.table("events"); try { - sender.table("events").uuidColumn("id", 1, 2); - return null; + sender.uuidColumn("id", 3, 4); + Assert.fail("recovered schema-required backlog fell back to legacy encoding"); } catch (LineSenderSchemaException e) { - interruptPreserved.set(Thread.currentThread().isInterrupted()); - return e; + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); } - }); - try { - Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); - writer.get().interrupt(); - LineSenderSchemaException error = result.get(5, TimeUnit.SECONDS); - Assert.assertNotNull(error); - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, error.getReason()); - Assert.assertTrue("writer interrupt status was cleared", interruptPreserved.get()); - } finally { - executor.shutdownNow(); - Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); - sender.close(); } }); } @@ -2273,6 +2309,42 @@ private void legacyVarcharTable(String table, String column, String... values) { eof(); } + private void legacyTimestampTable(String table, String column, long expectedMicros) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedMicros, i64()); + eof(); + } + + private void schemaTimestampTable( + String table, + int tableId, + long version, + String column, + long expectedNanos + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 1, column, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedNanos, i64()); + eof(); + } + private void legacyIpv4Table(String table, String column, int... values) { Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); Assert.assertEquals(QwpConstants.VERSION, u8()); @@ -3346,18 +3418,37 @@ private int varint() { } private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledgeData; private final List dataFrames = new ArrayList<>(); private final CountDownLatch pong = new CountDownLatch(1); private final AtomicInteger describeRequests = new AtomicInteger(); + private final int designatedIndex; private volatile int result; + private volatile boolean respondToDescribe = true; private final int tableId; private volatile long version; private long ackSequence; private SchemaHandler(int result, int tableId, long version) { + this(result, tableId, version, true); + } + + private SchemaHandler(int result, int tableId, long version, boolean acknowledgeData) { + this(result, tableId, version, acknowledgeData, 2); + } + + private SchemaHandler( + int result, + int tableId, + long version, + boolean acknowledgeData, + int designatedIndex + ) { this.result = result; this.tableId = tableId; this.version = version; + this.acknowledgeData = acknowledgeData; + this.designatedIndex = designatedIndex; } @Override @@ -3365,6 +3456,9 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (data.length >= QwpConstants.HEADER_SIZE && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { describeRequests.incrementAndGet(); + if (!respondToDescribe) { + return; + } ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); int nameLength = request.getShort(QwpConstants.HEADER_SIZE + 1 + Long.BYTES) & 0xffff; @@ -3377,6 +3471,9 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } dataFrames.add(data); notifyAll(); + if (!acknowledgeData) { + return; + } try { client.sendBinary(QwpWireTestUtils.buildAck(ackSequence++)); } catch (IOException e) { @@ -3445,7 +3542,7 @@ private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payloadLength) .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); if (result == QwpSchemaProtocol.RESULT_KNOWN) { - out.putInt(responseTableId).putLong(version).putShort((short) 2).putShort((short) 3) + out.putInt(responseTableId).putLong(version).putShort((short) designatedIndex).putShort((short) 3) .putShort((short) id.length).put(id).putInt(ColumnType.UUID).putShort((short) 0) .putShort((short) failed.length).put(failed).putInt(ColumnType.STRING).putShort((short) 0) .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 621ee509e..ba8d1433b 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7809,3 +7809,132 @@ The intentional, server-supported public-setter conversion catalogue is now complete. BINARY parser paths, LONG_ARRAY, non-ASCII CHAR text and malformed decimal metadata remain explicit server-contract decisions, not unfinished client converters. + +## D067 — ASYNC cold start uses best-effort legacy encoding + +Status: design accepted, implementation pending, 2026-09-15. Design revision +69. No production code or tests changed in this decision record. + +The full client-module diagnostic exposed a real contract conflict rather +than an array failure. `lazy_connect=true` selects the existing asynchronous +initial-connect policy, whose public contract says producers can buffer while +the server is unavailable. Schema activation instead made the first effective +write wait up to 30 seconds for negotiation and throw `SCHEMA_UNAVAILABLE`. +Preserving both mandatory cold-cache schema lookup and non-blocking offline +writes would require retaining original values for later conversion. That +would reintroduce the rejected send-time transformation model. + +Reuse the existing initial-connect choice instead. `OFF` makes one strict +foreground connection/negotiation attempt, while `SYNC` retries strictly within +its configured reconnect budget. Before any successful handshake, `ASYNC` +accepts a row immediately using the existing legacy conversion and flag-clear +framing, provided recovery has not already found schema-framed data. The first +effective operation pins the row's mode. A handshake completed during a partial +legacy row affects only the next row; buffered bytes are never relabelled, +rewritten or reconverted. + +This is a narrow startup exception, not a permanent schema switch. A successful +old-server handshake keeps subsequent rows legacy, but does not prohibit a +later one-way upgrade when another connection confirms schema support. Once +support is confirmed, cache misses and later outages retain the strict +30-second lookup/error contract and never fall back. A recovered extended +backlog also carries the sticky schema requirement before the fresh sender has +connected, so it overrides `ASYNC` fallback and waits for a supporting server. + +Source review confirms the simple wire path. The server rejects a schema flag +without negotiation but accepts flag-clear legacy frames on a negotiated +connection. The sender already pins a partial row and separates legacy and +schema-bound buffers at a row boundary. The cursor engine already records when +recovered frames require schema support. Implementation therefore needs no new +protocol field, raw-row store, deferred converter, persisted side file or +schema-specific configuration option. + +The availability cost is explicit. Pre-handshake rows use the whole legacy +conversion contract, not merely weaker validation. Handshake timing can change +successful stored values for cases such as sub-microsecond timestamps, +overflow and source-null normalization. A bad legacy row can also be accepted +locally and later receive a terminal NACK that stops the persisted queue; no +schema reply can repair its bytes. API documentation and release notes must +call this legacy conversion before confirmation and explain existing terminal +recovery, rather than promise deterministic schema-mode results. + +Required acceptance covers offline `ASYNC` startup followed by old and new +servers, handshake before the first row and during a partial row, old-to-new +upgrade, exact legacy/schema value differences, mixed replay across restart, +recovered schema-required backlog while offline, terminal rejection of a bad +startup row and a post-upgrade cache miss. Replace the stale facade recovery +expectation that a blocking first write returns before the test starts its +server; retain its observable construction, reconnect, write and read recovery +coverage under the new contract. + +## D068 — Integrate ASYNC fallback with one existing sticky latch + +Status: implemented and validated in the working tree, 2026-09-15. Design +revision 70. Not yet committed. + +Further source tracing reduced D067 to one producer-side decision in +`QwpWebSocketSender.bindingForEffectiveWrite()`. Preserve the existing partial- +row return first and call `ensureConnected()` as today. Before the current +bounded negotiation/lookup path, return the existing legacy binding result +when `initialConnectMode == ASYNC && !cursorEngine.requiresSchema()`, clearing +the per-call schema-freshness marker as on other legacy/in-progress paths. + +Do not consult `hasEverConnected()`. A fresh asynchronous sender and a sender +successfully connected to an old server both need legacy behavior, so +distinguishing them adds no decision value. The engine requirement already +becomes sticky when a supporting handshake succeeds and is restored directly +from recovered schema-framed data. It is volatile and published before the I/O +loop exposes the supporting connection, so it is also the correct race and +recovery authority. If it flips immediately after the producer reads false, +the current row is deliberately legacy and the existing row-boundary adoption +makes the next row schema-aware. + +No new state, helper abstraction, cache probe, schema mode, configuration key, +protocol field, raw-value owner, encoder path, persisted format, coordinator or +server change is justified. Existing retired-buffer and mixed-flush code owns +legacy-to-schema ordering, while the server already accepts flag-clear data on +a negotiated connection. + +The essential new coverage is narrow. Replace the asynchronous first-write +blocking test with one public test that buffers legacy data offline, connects +to a supporting server and verifies exact legacy then schema representations; +use a nanosecond `Instant` so the semantic difference is observable. Retarget +the existing timeout and interruption tests to a confirmed schema connection +whose describe response is withheld. Add one public recovered-schema-backlog +test proving that `ASYNC` still waits or fails instead of appending legacy data. +Reuse the existing facade late-server recovery, partial-row old-to-new upgrade, +mixed restart replay, real-server negotiated-legacy and terminal-NACK tests; +do not duplicate those fixtures. Final acceptance requires the complete client +suite and the standing released-binary compatibility matrix. + +The implementation matches the brief. `bindingForEffectiveWrite()` keeps its +in-progress-row path first, starts the existing I/O loop, then returns the +legacy binding for `ASYNC` only while the cursor engine does not require +schema. No production state, helper abstraction, coordinator path, encoder, +store-and-forward format, protocol field or server code changed. The public +`InitialConnectMode.ASYNC` documentation now states the legacy-before-support, +sticky-upgrade and recovered-backlog behavior. + +One fixture assumption was disproved during implementation: its `ts` column was +marked as the designated timestamp, so it could not expose the ordinary +timestamp representation difference. The fixture now reports no designated +column for that test. A nanosecond `Instant` then proves the actual contract: +the offline row contains legacy microseconds and the post-handshake row contains +schema-mode nanoseconds. + +Four focused new or changed tests pass. The full +`QwpSchemaSenderIntegrationTest` passes 72 tests and +`InitialConnectAsyncTest` passes seven. The complete client module passes 3,720 +tests with zero failures or errors and seven pre-existing skips. The current +server's five-test `QwpSchemaIdentityE2ETest` passes, including mixed +legacy/schema replay and legacy data on a negotiated connection. + +The released-binary matrix also passes: current client to QuestDB 10.0.1 uses +legacy conversion, client 1.3.9 to the current server remains legacy, and the +current client to current server enables schema mode. The packaged artifacts +used client SHA-256 +`74bbc706b69a53b89b84ce738865321b639d06b92200e1e1fa497e85c36e2e40` +and server SHA-256 +`38d597d80b12bb3c5ea13aa490b23df83061fb7299394b7c36a2c8227860627d`. +The completed full-client, build, server and compatibility runs used scratch +under `/mnt/pcie5`; `git diff --check` passes. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index d33ec6d37..b16d96109 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,13 +1,17 @@ # Schema-aware sender: schema-directed encoding -Status: implemented on the development branch, revision 68. Scope: QWP v1 over -WebSocket, with an automatically negotiated schema extension, legacy-server -compatibility and companion server changes. The committed baseline contains the -protocol, Sender integration and conversions through iteration 2.44. Iteration -2.45 (ranked DOUBLE-array identity) is locally validated. A released-binary -compatibility gate is implemented and locally green. The intentional, -server-supported public-setter conversion catalogue is complete. The explicit -server-contract boundaries below remain excluded rather than emulated. +Status: schema-directed encoding is implemented on the development branch; +design revision 70. Scope: QWP v1 over WebSocket, with an automatically +negotiated schema extension, legacy-server compatibility and companion server +changes. The committed baseline contains the protocol, Sender integration and +conversions through iteration 2.45. A released-binary compatibility gate is +implemented and locally green. The intentional, server-supported public-setter +conversion catalogue is complete. Revision 69 defines best-effort legacy +encoding during a genuinely asynchronous cold start; revision 70 reduces its +integration to one existing sticky schema requirement. That availability +change is implemented and validated in the working tree but is not yet +committed. The explicit server-contract boundaries below remain excluded +rather than emulated. ## Goal and contract @@ -40,14 +44,20 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.45 adds all public DOUBLE-array representations -to canonical DOUBLE-array targets of exactly the same rank. Its pre-iteration -baseline is client `47a37a6f` and server `c1f970220f`, whose submodule pins that -exact client revision. The client reuses the existing array buffer and encoder, -validates known rank before append, and pins the first effective rank for a -confirmed missing column. The intentional conversion inventory below is now -complete. This does not turn the separately documented unsafe or undefined -server paths into supported conversions. +Current checkpoint: iteration 2.46 implements revision 70's asynchronous +cold-start behavior in the working tree. Before schema support is known, an +`ASYNC` sender uses the legacy contract; after a supporting handshake, the +existing engine requirement makes schema mode sticky. Recovered schema-framed +backlog bypasses the legacy branch. The production change is one branch in +`bindingForEffectiveWrite()` plus API documentation; it adds no state, protocol +field, persisted format or send-time conversion. + +The committed baseline remains client `bc16c164` and server `02af9a6f31`, whose +submodule pins that exact client revision. Iteration 2.45 on that baseline adds +all public DOUBLE-array representations to canonical DOUBLE-array targets of +exactly the same rank. The intentional conversion inventory is complete. This +does not turn the separately documented unsafe or undefined server paths into +supported conversions. The standing compatibility gate now runs three real process combinations: current client against QuestDB 10.0.1, client 1.3.9 against the current server, @@ -625,19 +635,26 @@ response header selects legacy behavior, unless this sender has previously confirmed schema support. Explicit confirmation enables schema requests, schema-directed encoding and extended responses. An unrecognized response header is a negotiation error, not evidence of an old server; neither are -connection failures, timeouts or authorization failures. +connection failures, timeouts or authorization failures. Before any successful +upgrade, `ASYNC` may nevertheless choose the existing legacy encoding contract +for a new row. That is an explicit availability policy, not a claim that the +unreachable peer is an old server. The upgrade is one-way: once a sender confirms schema support, it assumes all subsequent connections support the extension too. Require confirmation on reconnect and background replay connections; never switch back to legacy mode. Moving an upgraded sender to an old server is unsupported. Retain pending data and report the capability mismatch under the existing reconnect policy. There -is no send-time transformation, metadata stripping or downgrade encoder. +is no send-time transformation, metadata stripping or downgrade encoder. A +successful connection to an old server does not permanently prevent upgrade: +a later connection that confirms schema support adopts schema mode at the next +row boundary. | Client | Server | Behavior | |---|---|---| | Existing client, no schema request | New server | Existing QWP v1 framing and behavior | | New client, not yet upgraded | Old server | Existing inferred-type QWP v1 behavior | +| New client, `ASYNC`, no completed handshake | Unknown server | Best-effort legacy encoding until a handshake selects the mode | | New client, always requests schema | Supporting server | QWP v1 with the schema extension; one-way upgrade | | New client, already upgraded | Old server | Unsupported endpoint; retain pending data, no downgrade | @@ -650,7 +667,9 @@ old/new endpoints are not supported for that sender after the upgrade. An existing legacy row finishes in legacy mode. The producer adopts schema mode at the next actual row boundary, seals completed legacy rows and replaces the inferred layout before binding new rows. Pending legacy blocks retain -their original types and flag-clear framing, even if encoded or sent later. +their original types and flag-clear framing, even if encoded or sent later. A +supporting server accepts those flag-clear blocks on a schema-negotiated +connection; negotiation is required only for schema-flagged data. The extension defines one stable conversion contract, backed by shared test vectors, rather than a separate conversion-version handshake or a dependency on @@ -661,25 +680,85 @@ raw-value encoding. Schema lookup failures never disable schema mode. ## Availability -In schema mode, required lookups use a fixed 30-second deadline. On the first -write, mode selection and its immediately following describe share one -end-to-end budget; a later cache-miss describe or forced refresh starts its own -30-second budget. This is not a public configuration surface. -Timeout, unavailable metadata or request saturation throws a lookup error -before accepting the value. It cancels any partial row but does not permanently -halt the sender. Do not add automatic retries or separate timeout settings for -the three lookup paths. +Reuse the existing initial-connection policy; do not add a schema-specific mode +or option: + +- `OFF` makes one foreground connection and negotiation attempt. Failure aborts + construction. +- `SYNC` retries connection and negotiation on the caller thread within the + configured reconnect budget. Exhaustion aborts construction. +- `ASYNC` returns before connecting. Until the first successful handshake, a + row whose first effective operation finds no negotiated mode is accepted + immediately under the existing legacy conversion and flag-clear framing. + This is best-effort schema use, not deferred validation: the client never + retains the original values for later conversion. + +The mode is pinned by the first effective operation. If the handshake completes +mid-row, that row remains legacy and schema mode can start only on the next row. +Consequently, handshake timing can change successful stored values as well as +error timing: the legacy and schema contracts deliberately differ for cases +such as sub-microsecond timestamps, overflow and source-null normalization. +Release notes and API documentation must describe this as legacy conversion +before confirmation, not merely skipped validation. + +The `ASYNC` exception applies only before the first successful handshake and +only when recovery has not already found schema-framed data. A recovered +schema-required backlog must negotiate schema support before accepting new +data; it must not append speculative legacy rows. If the first successful +server is old, rows remain legacy until a later connection confirms schema +support. Once support is confirmed, the sender never falls back to legacy, +including on reconnect, cache miss or lookup failure. + +After schema support is confirmed, required lookups use a fixed 30-second +deadline. A cache-miss describe or forced refresh starts its own 30-second +budget. This is not a public configuration surface. Timeout, unavailable +metadata or request saturation throws a lookup error before accepting the +value. It cancels any partial row but does not permanently halt the sender. Do +not add automatic retries or separate timeout settings for lookup paths. + +### Minimal client integration + +Change only `QwpWebSocketSender.bindingForEffectiveWrite()`. Keep its existing +in-progress-row fast path first, then ensure the existing I/O loop is started. +Before entering the current deadline, negotiation and lookup path, select +legacy behavior with this condition: + +```java +if (initialConnectMode == Sender.InitialConnectMode.ASYNC + && !cursorEngine.requiresSchema()) { + schemaResolutionFresh[0] = false; + return null; +} +``` + +`CursorSendEngine.requiresSchema()` is the existing sticky authority. It is +false for a fresh asynchronous sender and after a successful old-server +handshake. It becomes true before publication after a supporting handshake, +and recovery initializes it to true when persisted frames require schema +support. That single value therefore covers startup, old-server compatibility, +one-way upgrade and recovery. Do not also consult `hasEverConnected()` and do +not add another negotiated-mode field. + +A supporting handshake may set the requirement immediately after the producer +observes false. That race is harmless and intentional: the current row has +already selected legacy behavior, while the next row observes the sticky true +value and enters schema mode. The existing partial-row check, retired table +buffers and mixed-frame flush preserve that boundary without a new lock or row +state. + +No change belongs in `QwpSchemaCoordinator`, `CursorWebSocketSendLoop`, +`CursorSendEngine`, the encoder, store-and-forward format or server. Update the +`ASYNC` API documentation and the outdated method comment that currently says +an unavailable first connection can never select legacy encoding. Known schemas can be used offline; unfamiliar tables require a live lookup. Reconnect invalidates the lookup cache for future row starts, but does not reset an upgraded sender to legacy mode, invalidate an already pinned row or rewrite -buffered and persisted blocks. Legacy mode retains existing offline buffering. - -Asynchronous construction may remain, but the first write waits for the initial -successful negotiation to select the mode within a bounded deadline. An offline -or unreachable server is not assumed to be old. In schema mode, required lookup -must also complete before accepting the value; in legacy mode, no lookup is -needed. This changes cold-start offline behavior, not the minimum server version. +buffered and persisted blocks. Legacy mode and the pre-handshake `ASYNC` +exception retain existing offline buffering. A bad speculative legacy row may +therefore be rejected only after connection and can stop the persisted queue +under the existing terminal-NACK policy; metadata cannot repair or reconvert +it. This is the cost of choosing asynchronous startup. ## Frame layout and replay compatibility @@ -1776,10 +1855,19 @@ They do not change the compatibility contract. LONG-array casting. The current server checks rank but can build storage from the cursor's element type, so accepting a non-DOUBLE target would reproduce a server validation gap rather than a supported conversion. - -These opportunities do not justify a per-setter opt-out, raw-value fallback or -send-time transformation. The intentional conversion catalogue is complete on -the unreleased development branch. Release review must still verify the standing +28. **Reuse initial-connect policy only at the cold-start boundary.** Let + `ASYNC` create ordinary legacy rows until the first handshake, then use the + existing row-boundary transition to adopt schema mode. Keep `OFF` and + `SYNC` strict, and let a recovered schema-required backlog override the + startup exception. Do not add raw-value storage, a deferred converter, a + second availability option or a downgrade path after schema confirmation. + Extending fallback to later cache misses would turn a connection option + into a permanent validation switch and is intentionally excluded. + +Outside the explicit pre-handshake `ASYNC` exception, these opportunities do +not justify a per-setter opt-out, raw-value fallback or send-time +transformation. The intentional conversion catalogue is complete on the +unreleased development branch. Release review must still verify the standing gates and explicitly accept or resolve the server-contract boundaries above; they must not be hidden behind fallback behavior. @@ -1848,7 +1936,7 @@ DATE grammar sequence and selects target-native DATE storage. Calendar and timezone mechanics are reused from the existing fixed timestamp parser; no general date-format framework was added. -Iteration 2.45 is complete locally: every public DOUBLE-array overload selects +Iteration 2.45 is committed: every public DOUBLE-array overload selects the existing DOUBLE_ARRAY wire representation only when the server's canonical element type and rank match. The N-dimensional wrapper exposes its current rank; confirmed missing columns pin their first effective rank. Exact-wire tests cover @@ -1862,6 +1950,20 @@ BINARY-to-parser behavior, LONG_ARRAY validation, non-ASCII CHAR text output or malformed decimal metadata. Until then, rejecting those paths is simpler and safer than copying an accidental server behavior into the client. +Iteration 2.46 implements the revision-70 availability change with the single +`ASYNC && !cursorEngine.requiresSchema()` branch above; the existing sticky +requirement is the recovery guard. Public tests now verify exact legacy +microseconds before connection and schema nanoseconds after a supporting +handshake, timeout and interruption after schema support is confirmed, and no +legacy fallback when recovery finds schema-framed backlog. Existing partial-row +upgrade and mixed-replay coverage remains unchanged. + +Acceptance is green: the complete client suite passes 3,720 tests with zero +failures or errors, the five-test real-server schema identity/replay suite +passes, and the released-binary matrix passes current-client/old-server, +old-client/current-server and current-client/current-server combinations. This +is an availability correction, not another conversion family. + Every slice keeps the existing observable-contract rules below: public API, exact wire, SQL result, partial-row rollback and relevant recovery behavior. Run canonical and packaged-client gates, the server gate against its pinned @@ -1960,19 +2062,27 @@ without claiming general exactly-once delivery. ## Acceptance checks -- New clients always request the extension. Before upgrade, an old server's - absent confirmation selects existing inferred-type behavior, with unchanged +- New clients always request the extension. A successful old-server handshake + without confirmation selects existing inferred-type behavior, with unchanged wire framing, server conversions, missing-value semantics and SF delivery. - Supporting servers enable schema mode automatically; no option disables it. -- Failed handshakes, timeouts and invalid confirmation are not treated as old - servers. Async startup waits for mode selection and cannot bypass a required - first-use lookup. + Supporting servers enable schema mode automatically; no option disables it + after confirmation. +- With no completed handshake and no recovered schema-required backlog, + `ASYNC` accepts flag-clear legacy rows without waiting. Verify offline startup + followed by both old and supporting servers, handshake before the first row, + handshake during a partial row, and a later old-to-new upgrade. Pin legacy + conversion results where they differ from schema mode, including timestamp + precision, overflow and source-null handling. +- Failed handshakes, timeouts and invalid confirmation are not classified as + old servers. `OFF` and `SYNC` remain strict. Recovered schema-framed backlog + overrides `ASYNC` fallback and sends no data until schema support is confirmed. - Existing clients against new servers see byte-compatible QWP v1 behavior, with no unsolicited schema messages or extended responses. The base protocol version remains 1 for both legacy and extended traffic. - The persisted flag selects table-header parsing. Exercise mixed legacy and extended SF replay on a negotiated connection, and reject flagged frames on - unnegotiated connections before processing data. + unnegotiated connections before processing data. Restart with a mixed backlog + and prove its original bytes, order and schema requirement are preserved. - Upgrade from an old to a supporting server finishes any partial legacy row unchanged and adopts schema mode at the next row boundary, including without another `table()` call. Pending legacy blocks and SF remain unchanged and @@ -1982,6 +2092,9 @@ without claiming general exactly-once delivery. advancement. The requirement survives reconnect even with a legacy-only backlog; recovery detects pending extended SF before sending a legacy prefix. Legacy-only recovery by a fresh sender remains compatible with old servers. +- A bad pre-handshake `ASYNC` legacy row is rejected through the existing + server-error and terminal-SF path; it is never silently repaired, converted + or acknowledged. Document the resulting queue-stop and recovery behavior. - The downgrade procedure drains every pending extended slot with a supporting client under the configured ACK policy. Interrupted or timed-out draining leaves recoverable data and does not qualify as safe to downgrade. @@ -2026,8 +2139,9 @@ without claiming general exactly-once delivery. frame order, transaction boundaries, backpressure and completed rows on error. - Auto-create, multi-table writes, cumulative ACKs, deferred commits, delayed sends and describe/ACK interleaving preserve schema feedback and watermarks. -- Lookup timeout, offline cache misses, eviction, reconnect and shutdown neither - bypass required resolution nor invalidate pinned data or strand callers. +- After schema confirmation, lookup timeout, offline cache misses, eviction, + reconnect and shutdown neither bypass required resolution nor invalidate + pinned data or strand callers. `ASYNC` does not enable later fallback. - Unsupported target types fail explicitly when used; unrelated columns with unknown types do not block supported writes. The million-entry cache grows on demand and obsolete pinned metadata is released. From 55fbc1f2c50a337f5cab19836b72e8862a160312 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 15 Sep 2026 10:55:26 +0200 Subject: [PATCH 16/51] docs(qwp): update schema-aware sender status --- design/schema-aware-sender.md | 46 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index b16d96109..344935d59 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -3,15 +3,15 @@ Status: schema-directed encoding is implemented on the development branch; design revision 70. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server -changes. The committed baseline contains the protocol, Sender integration and -conversions through iteration 2.45. A released-binary compatibility gate is -implemented and locally green. The intentional, server-supported public-setter -conversion catalogue is complete. Revision 69 defines best-effort legacy -encoding during a genuinely asynchronous cold start; revision 70 reduces its -integration to one existing sticky schema requirement. That availability -change is implemented and validated in the working tree but is not yet -committed. The explicit server-contract boundaries below remain excluded -rather than emulated. +changes. The committed implementation contains the protocol, Sender integration +and conversions through iteration 2.46 at client `59ade916` and server +`ea019a99f3`, whose submodule pins that exact client revision. A released-binary +compatibility gate is implemented and locally green. The intentional, +server-supported public-setter conversion catalogue is complete. Revision 69 +defines best-effort legacy encoding during a genuinely asynchronous cold start; +revision 70 reduces its integration to one existing sticky schema requirement. +That availability change is committed and locally validated. The explicit +server-contract boundaries below remain excluded rather than emulated. ## Goal and contract @@ -44,20 +44,20 @@ server-side failures can still reject a batch. ## Implementation status -Current checkpoint: iteration 2.46 implements revision 70's asynchronous -cold-start behavior in the working tree. Before schema support is known, an -`ASYNC` sender uses the legacy contract; after a supporting handshake, the -existing engine requirement makes schema mode sticky. Recovered schema-framed -backlog bypasses the legacy branch. The production change is one branch in -`bindingForEffectiveWrite()` plus API documentation; it adds no state, protocol -field, persisted format or send-time conversion. - -The committed baseline remains client `bc16c164` and server `02af9a6f31`, whose -submodule pins that exact client revision. Iteration 2.45 on that baseline adds -all public DOUBLE-array representations to canonical DOUBLE-array targets of -exactly the same rank. The intentional conversion inventory is complete. This -does not turn the separately documented unsafe or undefined server paths into -supported conversions. +Current checkpoint: iteration 2.46 is committed as client `59ade916` and pinned +by server `ea019a99f3`. It implements revision 70's asynchronous cold-start +behavior. Before schema support is known, an `ASYNC` sender uses the legacy +contract; after a supporting handshake, the existing engine requirement makes +schema mode sticky. Recovered schema-framed backlog bypasses the legacy branch. +The production change is one branch in `bindingForEffectiveWrite()` plus API +documentation; it adds no state, protocol field, persisted format or send-time +conversion. + +Iteration 2.45 at client `bc16c164` and server `02af9a6f31` added all public +DOUBLE-array representations to canonical DOUBLE-array targets of exactly the +same rank. The intentional conversion inventory is complete. This does not turn +the separately documented unsafe or undefined server paths into supported +conversions. The standing compatibility gate now runs three real process combinations: current client against QuestDB 10.0.1, client 1.3.9 against the current server, From 31111e4a352bafe2308a531f0d3acec1ef313a89 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 15 Sep 2026 13:35:19 +0200 Subject: [PATCH 17/51] fix(qwp): restore char-to-text conversion --- .../qwp/protocol/QwpSchemaBinding.java | 30 ++++++++- .../QwpSchemaSenderIntegrationTest.java | 4 +- .../protocol/QwpSchemaBindingCharTest.java | 34 ++++++++++ design/schema-aware-sender-journal.md | 35 ++++++++++ design/schema-aware-sender.md | 66 ++++++++++++------- 5 files changed, 142 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index a12f3c11c..ee74c58e5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -82,6 +82,7 @@ public final class QwpSchemaBinding { private final QwpSchemaResponse schema; private final String tableName; private Decimal256 decimalTextScratch; + private StringSink charTextSink; private int[] floatingTextExponentScratch; private StringSink floatingTextSink; private StringSink numericTextSink; @@ -701,9 +702,32 @@ public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { public QwpSchemaBinding charColumn(CharSequence name, char value) { buffer.requireSchemaBinding(this); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "CHAR", ColumnType.CHAR); - if (column != null) { - column.addShort((short) value); + int index = targetIndex(name, "CHAR"); + int targetType = targetType(index, ColumnType.CHAR); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "CHAR", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.CHAR: + column.addShort((short) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + if (Character.isSurrogate(value)) { + throw error(INVALID_VALUE, name, "CHAR", targetType, "surrogate cannot be converted to text"); + } + StringSink sink = charTextSink; + if (sink == null) { + charTextSink = sink = new StringSink(1); + } else { + sink.clear(); + } + sink.put(value); + column.addString(sink); + break; + default: + throw unsupported(name, "CHAR", targetType, "conversion is not implemented"); } return this; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 1f60e5e72..90be992f7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -774,7 +774,7 @@ public void testStringToCharRollbackAndRecoveryWithoutTableReselection() throws } @Test - public void testVarcharGenerationRemainsPinnedBeforeNativeCharRebind() throws Exception { + public void testVarcharGenerationRemainsPinnedBeforeCharRebind() throws Exception { assertMemoryLeak(() -> { LongTextSchemaHandler handler = new LongTextSchemaHandler(411, 421, ColumnType.VARCHAR); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { @@ -783,7 +783,7 @@ public void testVarcharGenerationRemainsPinnedBeforeNativeCharRebind() throws Ex handler.version = 422; sender.table("events"); try { - sender.charColumn("value", 'B'); + sender.binaryColumn("value", new byte[]{1}); Assert.fail("expected target-changing schema refresh"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java index c248f78fb..f0ec81e52 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java @@ -29,6 +29,33 @@ public class QwpSchemaBindingCharTest { private static final String CORPUS = "/io/questdb/client/cutlass/qwp/char-conversions.tsv"; private static final String HEADER = "# case_id\tinput_kind\tutf16_hex\texpected_char\texpected_null"; + @Test + public void testCharToTextUsesTargetNativeUtf8() { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", targetType))); + binding.charColumn("value", 'A'); + buffer.nextRow(); + binding.charColumn("value", '\u03a9'); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + Reader reader = tableHeader(encoder, size, 2, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(3, reader.i32()); + Assert.assertEquals('A', reader.u8()); + Assert.assertEquals(0xce, reader.u8()); + Assert.assertEquals(0xa9, reader.u8()); + Assert.assertEquals(size, reader.position); + } + } + } + @Test public void testCorpusUsesExactNativeCharWire() throws Exception { InputStream stream = getClass().getResourceAsStream(CORPUS); @@ -158,6 +185,13 @@ public void testUnsupportedAndDesignatedTargetsRemainRejected() { assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.charColumn("ts", 'A')); } + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", targetType))); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.charColumn("value", '\ud800')); + } + } } private static byte[] column(String name, int type) { diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index ba8d1433b..e811b09de 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7938,3 +7938,38 @@ and server SHA-256 `38d597d80b12bb3c5ea13aa490b23df83061fb7299394b7c36a2c8227860627d`. The completed full-client, build, server and compatibility runs used scratch under `/mnt/pcie5`; `git diff --check` passes. + +## D069 — Restore CHAR-to-text compatibility with valid UTF-8 + +Status: implemented and locally validated in the working tree, 2026-09-15. +Design revision 71. Not yet committed. + +The complete server E2E classes exposed a conversion omitted by the earlier +hand-picked gates. On a known STRING or VARCHAR column, `charColumn` requested +only a CHAR target and therefore threw `UNSUPPORTED_FEATURE`. The existing +server accepts CHAR for both text targets, so mandatory schema mode turned a +previously valid public call into a local failure. The design had documented +the pair as an unresolved boundary, but that classification was not acceptable +for an always-on feature. + +Resolve the actual target in the existing `charColumn` path. Keep exact CHAR +storage unchanged. For STRING and VARCHAR, append the character through one +reused single-character sink into the existing target-native VARCHAR buffer. +This adds no conversion framework, protocol field, sender state, replay path, +server change or per-row allocation. + +Do not reproduce the server formatter's non-ASCII VARCHAR truncation. A valid +non-surrogate Java CHAR is encoded as valid UTF-8 for both text targets. A lone +UTF-16 surrogate cannot be represented as a standalone Unicode character, so +the setter reports `INVALID_VALUE` instead of silently replacing or corrupting +it. Native CHAR targets continue to preserve every 16-bit code unit. + +The focused component test passes four tests and checks exact target wire bytes +for ASCII and non-ASCII input plus surrogate rejection. The complete client +module passes 3,721 tests with no failures or errors and seven skips. After +installing that client artifact, the complete 138-test real-server E2E class +passes both existing regressions (`testCoercionToString` and +`testCoercionToVarchar`) and a new non-ASCII SQL assertion. Its remaining two +failures and 20 errors are the previously catalogued DATE/GEOHASH wire issues +and stale legacy rejection expectations, down from two failures and 22 errors +before this change. All test scratch owned by this work is under `/mnt/pcie5`. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 344935d59..5ad393c18 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1,7 +1,7 @@ # Schema-aware sender: schema-directed encoding Status: schema-directed encoding is implemented on the development branch; -design revision 70. Scope: QWP v1 over WebSocket, with an automatically +design revision 71. Scope: QWP v1 over WebSocket, with an automatically negotiated schema extension, legacy-server compatibility and companion server changes. The committed implementation contains the protocol, Sender integration and conversions through iteration 2.46 at client `59ade916` and server @@ -10,8 +10,10 @@ compatibility gate is implemented and locally green. The intentional, server-supported public-setter conversion catalogue is complete. Revision 69 defines best-effort legacy encoding during a genuinely asynchronous cold start; revision 70 reduces its integration to one existing sticky schema requirement. -That availability change is committed and locally validated. The explicit -server-contract boundaries below remain excluded rather than emulated. +Revision 71 restores CHAR-to-STRING/VARCHAR compatibility with target-native +UTF-8 encoding. That conversion is implemented and locally validated in the +working tree. The explicit server-contract boundaries below remain excluded +rather than emulated. ## Goal and contract @@ -44,6 +46,15 @@ server-side failures can still reject a batch. ## Implementation status +Current working tree: iteration 2.47 implements CHAR-to-STRING/VARCHAR. It +preserves native CHAR storage, encodes non-surrogate characters directly as +target-native VARCHAR wire data for either text target, and rejects a lone +UTF-16 surrogate locally. All 3,721 client tests pass. The complete 138-test +server E2E class removes both CHAR regressions and passes the dedicated Unicode +case; its remaining 2 failures and 20 errors are the previously catalogued +DATE/GEOHASH and stale legacy-expectation cases. D069 records the decision and +evidence. + Current checkpoint: iteration 2.46 is committed as client `59ade916` and pinned by server `ea019a99f3`. It implements revision 70's asynchronous cold-start behavior. Before schema support is known, an `ASYNC` sender uses the legacy @@ -416,12 +427,13 @@ diagnostic now has 26 errors, removing only its two timestamp-to-text cases. No repeatable native timestamp-write regression is detected by the performance guard; this does not measure formatter throughput. D044 records the evidence. -Iteration 2.29 accepts native CHAR and STRING-to-CHAR using existing two-byte +Iteration 2.29 accepted native CHAR and STRING-to-CHAR using existing two-byte storage. Native values retain all 16 bits. Text takes the first BMP character, ignoring the remainder; empty/NUL/supplementary-first text becomes code unit zero, while malformed UTF-16 becomes '?', matching existing client replacement and server decoding. Java null is bitmap-null; present zero remains wire-present -but reads as SQL NULL. CHAR-to-text remains separate missing work. Validation +but reads as SQL NULL. CHAR-to-text remained separate at that checkpoint and +is implemented by iteration 2.47. Validation passes 452 client tests plus two packaging checks, 59 focused server tests, 323 regressions and three 86-test seeded repeats. All 65,549 public differential checks match the server. The unchanged original 333-test diagnostic has 24 @@ -1115,7 +1127,7 @@ non-null conversion. Known compatibility exceptions above still apply. | `timestampColumn(long, unit)` (TIMESTAMP or TIMESTAMP_NANOS) | Timestamps, text | None | | `timestampColumn(Instant)` (currently TIMESTAMP micros in legacy Sender) | Timestamps, preserving nanos for the schema-mode nano target; text | None | | `uuidColumn` (UUID) | UUID, text | None | -| `charColumn` (CHAR) | CHAR | Text; non-ASCII text output needs the decision below | +| `charColumn` (CHAR) | CHAR, text for non-surrogate values | None | | `ipv4Column(int)` and `ipv4Column(CharSequence)` (IPv4; text parsed locally) | IPv4, text | None | | `long256Column` (LONG256) | LONG256, text | None | | `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | @@ -1287,13 +1299,13 @@ unsafe behaviour merely because it is reachable is not the recommendation. Thus the existing-target route is not proof of correct LONG_ARRAY ingestion or cross-element conversion. Resolve the server validation/compatibility gap rather than inventing client-side element casts to cover it. -3. **CHAR-to-text is not uniformly Unicode-safe.** CHAR-to-STRING writes the - UTF-16 code unit; CHAR-to-VARCHAR calls `Utf8StringSink.putAscii(char)`, which - truncates it to one byte. Non-ASCII values can therefore produce corrupted - VARCHAR bytes. A surrogate code unit in STRING also cannot be preserved by - the binding's normal VARCHAR wire encoding. Keep these value-level limits - visible; correct UTF-8 output would require a deliberate compatibility - decision, not an undocumented change disguised as parity. +3. **CHAR-to-text uses valid UTF-8, not the server's unsafe VARCHAR path.** The + legacy server formatter truncates a non-ASCII CHAR when targeting VARCHAR. + Iteration 2.47 deliberately fixes that behavior in schema mode by encoding + any non-surrogate CHAR as target-native UTF-8 for both STRING and VARCHAR. + A lone UTF-16 surrogate has no valid standalone UTF-8 representation and is + rejected locally instead of being replaced or corrupted. Native CHAR writes + still preserve all 16 bits. 4. **Designated timestamp fallback is not a conversion catalogue.** Its special branch can pass a fixed-width cursor to timestamp storage without the ordinary-column source-type allowlist. Do not derive new public casts from @@ -1770,10 +1782,10 @@ They do not change the compatibility contract. D050 applies this rule. 13. **Implement intentional conversions, not every reachable cast.** The server audit is evidence, not a requirement to reproduce parser accidents or - validation gaps. Keep BINARY-to-parser conversions, LONG_ARRAY, non-ASCII - CHAR-to-VARCHAR and malformed decimal metadata out until their server - contracts are deliberately resolved. Matching a server bug is not - compatibility. + validation gaps. Keep BINARY-to-parser conversions, LONG_ARRAY and malformed + decimal metadata out until their server contracts are deliberately resolved. + For CHAR-to-VARCHAR, D069 applies this rule by emitting correct UTF-8 instead + of copying the server's one-byte truncation bug. 14. **Keep the schema deadline fixed.** Initial negotiation and its first describe share one 30-second end-to-end budget; later cache-miss describes and forced refreshes each get their own 30-second budget. Do not add public @@ -1944,11 +1956,21 @@ ranks 1, 2, 3 and 32, raw floating-point bits, empty shapes, duplicate/rollback behavior and generation changes. Real-server tests cover public ranks 1 through 4, stored shapes, rank rejection and partial-row recovery. -There is no remaining intentional client conversion slice. Further datatype -work must start by resolving one of the explicit server-contract boundaries: -BINARY-to-parser behavior, LONG_ARRAY validation, non-ASCII CHAR text output or -malformed decimal metadata. Until then, rejecting those paths is simpler and -safer than copying an accidental server behavior into the client. +There is no remaining intentional client conversion slice after iteration +2.47. Further datatype work must start by resolving one of the explicit +server-contract boundaries: BINARY-to-parser behavior, LONG_ARRAY validation +or malformed decimal metadata. Until then, rejecting those paths is simpler +and safer than copying accidental server behavior into the client. + +Iteration 2.47 restores CHAR-to-STRING/VARCHAR compatibility. The binding +selects the actual schema target, retains the existing two-byte CHAR path, and +uses one reusable single-character sink to append target-native VARCHAR wire +data for STRING or VARCHAR. Valid non-ASCII characters are encoded as UTF-8; +lone surrogates fail locally. No protocol, sender state, replay path or server +production change is added. The complete client suite passes 3,721 tests with +seven skips. The complete server E2E class runs 138 tests: both former CHAR +regressions and the new Unicode assertion pass, leaving 2 failures and 20 errors +from the separately tracked DATE/GEOHASH and stale legacy-expectation cases. Iteration 2.46 implements the revision-70 availability change with the single `ASYNC && !cursorEngine.requiresSchema()` branch above; the existing sticky From acf5ac0ecd7c71512d3e564ad035a402d4a56c83 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 15 Sep 2026 14:53:46 +0200 Subject: [PATCH 18/51] docs(qwp): record schema-aware E2E migration --- design/schema-aware-sender-journal.md | 46 +++++++++++++++++++ ...ma-aware-sender-remaining-failures-plan.md | 30 ++++++++++++ 2 files changed, 76 insertions(+) diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index e811b09de..300a5e498 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -7973,3 +7973,49 @@ passes both existing regressions (`testCoercionToString` and failures and 20 errors are the previously catalogued DATE/GEOHASH wire issues and stale legacy rejection expectations, down from two failures and 22 errors before this change. All test scratch owned by this work is under `/mnt/pcie5`. + +## D070 — Align complete server E2Es with mandatory schema validation + +Status: accepted, 2026-09-15. The associated QuestDB changes are test-only. + +The complete server test classes confirmed that the remaining failures were +stale test contracts and unstable test-only buffer handles, not missing client +conversions. Twenty-two Sender cases expected a server rejection for invalid +input that a schema-aware client must now reject before encoding. Their shared +helper now accepts only typed, non-retryable local schema errors and verifies +row recovery. A separate strict helper retains the server-NACK contract for +server-owned failures. No helper may accept either outcome interchangeably. + +The deferred-commit case now states the actual ownership rule: a local schema +error cancels only the current partial row. It cannot ask the server to roll +back already completed deferred rows because the invalid row never reached the +server. Empty column names, STRING input for DOUBLE, and timestamp overflow +likewise assert local structured errors and subsequent valid writes. + +DATE and GEOHASH integration fixtures exposed a separate test bug. They held a +test-only column object across rows, but schema installation can replace the +sender's active table buffer at the row boundary. Public integration tests now +write through the public row API. The six deliberately low-level tests keep +their raw coverage but reacquire the active column on every row. This adds no +production state, compatibility path, defensive check or abstraction. + +The two real-server asynchronous in-flight/fragmentation tests no longer try to +force a type mismatch through a schema-aware public API. They now send a valid +public row to a non-WAL table: local validation succeeds and the server returns +the server-owned terminal NACK whose propagation those tests exercise. The +immediate type-mismatch test was removed because that state is unreachable after +discovery and its useful NACK behavior duplicates the existing non-WAL test. +Deterministic client tests continue to cover broader asynchronous NACK and +terminal-error delivery. + +Validation passes the complete 138-test `QwpSenderE2ETest`, the complete +117-test `QwpWebSocketSenderReceiverTest` with one existing skip, and all six +`QwpSenderLowLevelTest` cases. The combined schema-aware server gate passes 480 +tests with zero failures or errors and one existing skip. The complete client +module passes 3,721 tests with zero failures or errors and seven existing skips. +The two asynchronous server-rejection cases pass three times each, including +their fragmented-transport variant. Final server scratch is under +`/mnt/pcie5/qwp-schema-aware-tests/final-repeat`; the final complete client run +uses `/mnt/pcie5/qwp-schema-aware-tests/client-final`, with no `/tmp` paths in +its 295 current Surefire reports. No client or server production source changed +in this decision. diff --git a/design/schema-aware-sender-remaining-failures-plan.md b/design/schema-aware-sender-remaining-failures-plan.md index f38761324..60ee00155 100644 --- a/design/schema-aware-sender-remaining-failures-plan.md +++ b/design/schema-aware-sender-remaining-failures-plan.md @@ -569,6 +569,36 @@ delete the test, loosen its expected error, or silently count a renamed/removed method as fixed. Readiness requires every baseline case's behavior to remain represented. +## Post-ledger whole-class cleanup — 2026-09-15 + +Running the complete server Sender classes after the 47-method ledger exposed +two kinds of test debt, not more missing conversions: + +- Twenty-two `QwpSenderE2ETest` cases still expected the server to reject input + that mandatory schema mode now rejects locally. They now require a typed, + non-retryable local schema error and cancel the partial row. The separate + helper for server-owned failures accepts only a server NACK. +- DATE and GEOHASH fixtures retained a test-only column handle across rows even + though schema discovery may replace the active table buffer after a row. + Public API tests now use the public row API. The six intentionally low-level + tests remain low-level and reacquire the active column for every row. + +The two asynchronous in-flight/fragmentation tests now use a public write to a +non-WAL table: schema validation succeeds, then the server returns the real +server-owned NACK those tests are meant to propagate. The immediate type- +mismatch test was removed because that path is no longer reachable after +discovery and its useful server-NACK behavior duplicates the existing non-WAL +test. Deterministic client tests retain broader terminal-error coverage. This is +a test-contract migration only: no client or server production code changed. + +The complete `QwpSenderE2ETest` passes 138 tests, the complete +`QwpWebSocketSenderReceiverTest` passes 117 with one existing skip, and the +complete `QwpSenderLowLevelTest` passes six. The combined schema-aware server +gate passes 480 tests with zero failures or errors and one existing skip. The +complete client module passes 3,721 tests with zero failures or errors and seven +existing skips. The two asynchronous server-rejection cases pass three times +each, including their fragmented-transport variant. + ## Definition of done and remaining release work Done for this plan: all 47 baseline cases are represented and pass, the original From 67467dc9ee673c3088ee140d5e8b2934a1545926 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 15 Sep 2026 16:47:31 +0200 Subject: [PATCH 19/51] Use one Ryu double formatter in the client The client carried two double-to-text implementations that disagreed on about 0.3% of values. Numbers.append() used the pre-Ryu algorithm and served the legacy ILP path, while QwpSchemaDoubleFormatter wrapped a private copy of the server's Ryu for schema-aware QWP writes. The same double therefore reached the server as two different strings, depending only on which transport carried it. Numbers.append() now formats through Ryu, copied from the server so that client and server produce identical text. RyuDouble moves to io.questdb.client.std as the single copy, and QwpSchemaBinding calls Numbers.append() directly. QwpSchemaRyuDouble and QwpSchemaDoubleFormatter are gone. RyuDouble calls Math.multiplyHigh, which Java 8 lacks, so Compat gains a multiplyHigh shim: the intrinsic on Java 9+, and a hand-rolled equivalent on Java 8. A differential over 20 million random pairs plus edge cases shows the fallback matches the intrinsic exactly. Retiring the old formatter also retires the FdBig bridge and the jdk.internal.math export flags it needed. The multi-release jar stays, because Compat still supplies currentPid() and onSpinWait() to five other callers. JarPackagingIT now guards the Compat variants rather than the FdBig ones, and still executes the formatter out of the packaged jar on both JDKs. This also fixes two Java 8 breaks that predate the change and that a modern JDK hides. QwpSchemaRyuDouble called Math.multiplyHigh from a shared source root, and QwpSchemaFeedbackResponseTest chained putShort() off position(), which returns Buffer rather than ByteBuffer on Java 8. Both would have failed the Java 8 CI job. The legacy ILP path changes its wire text for roughly 0.3% of doubles. Ryu emits the shortest form that still reads back to the same value, so the output is never longer and never loses precision; over 3 million random values both formatters round-trip to identical doubles. One NumbersTest expectation encoded the old extra digit and is updated. Verified on JDK 8 and JDK 25: 3721 unit tests and the packaging ITs pass on both, and the shared floating-to-text corpus matches on all 40 rows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BDP5ZVN8xijRp9LhQoreMf --- core/pom.xml | 41 +- .../qwp/protocol/QwpSchemaBinding.java | 4 +- .../protocol/QwpSchemaDoubleFormatter.java | 103 ---- .../java/io/questdb/client/std/Numbers.java | 457 ++++-------------- .../RyuDouble.java} | 18 +- .../java11/io/questdb/client/std/Compat.java | 28 +- .../java11/io/questdb/client/std/FdBig.java | 82 ---- .../java8/io/questdb/client/std/Compat.java | 24 +- .../java8/io/questdb/client/std/FdBig.java | 81 ---- .../client/QwpSchemaFeedbackResponseTest.java | 4 +- .../client/test/std/DoubleFormatSmoke.java | 4 +- .../client/test/std/JarPackagingIT.java | 29 +- .../questdb/client/test/std/NumbersTest.java | 5 +- 13 files changed, 163 insertions(+), 717 deletions(-) delete mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java rename core/src/main/java/io/questdb/client/{cutlass/qwp/protocol/QwpSchemaRyuDouble.java => std/RyuDouble.java} (98%) delete mode 100644 core/src/main/java11/io/questdb/client/std/FdBig.java delete mode 100644 core/src/main/java8/io/questdb/client/std/FdBig.java diff --git a/core/pom.xml b/core/pom.xml index bcef13053..965126cee 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -196,7 +196,7 @@ - @@ -222,7 +222,7 @@ + so the Compat shim compiles on each JDK. --> org.codehaus.mojo build-helper-maven-plugin @@ -450,8 +450,8 @@ + Java 9+ Compat shim and module-info.class into the jar ROOT, which + breaks Java 8 consumers. Refuse to publish from any other JDK. --> org.apache.maven.plugins maven-enforcer-plugin @@ -524,13 +524,13 @@ 11 11 questdb - --add-exports - java.base/jdk.internal.math=io.questdb.client - - --add-exports - java.base/jdk.internal.math=io.questdb.client + + -Xlint:none + -Xlint:none + -quiet + -quiet nothing-to-exclude-dummy-value-include-all-java11plus nothing-to-exclude-dummy-value-include-all-java11plus ${javac.target} @@ -562,8 +562,7 @@ -Xlint:none -Xlint:none - + -quiet -quiet + org.apache.maven.plugins maven-antrun-plugin @@ -621,8 +620,6 @@ the jar is broken on Java 9+ again (JarPackagingIT cross-checks the packaged jar against the source root) --> - - diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index ee74c58e5..9f465b446 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -83,7 +83,6 @@ public final class QwpSchemaBinding { private final String tableName; private Decimal256 decimalTextScratch; private StringSink charTextSink; - private int[] floatingTextExponentScratch; private StringSink floatingTextSink; private StringSink numericTextSink; private Utf8StringSink timestampTextSink; @@ -1416,11 +1415,10 @@ private CharSequence formatFloating(double value) { StringSink sink = floatingTextSink; if (sink == null) { floatingTextSink = sink = new StringSink(24); - floatingTextExponentScratch = new int[1]; } else { sink.clear(); } - QwpSchemaDoubleFormatter.append(sink, value, floatingTextExponentScratch); + Numbers.append(sink, value); return sink; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java deleted file mode 100644 index 30557a26a..000000000 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaDoubleFormatter.java +++ /dev/null @@ -1,103 +0,0 @@ -/*+***************************************************************************** - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * Licensed under the Apache License, Version 2.0 - ******************************************************************************/ -package io.questdb.client.cutlass.qwp.protocol; - -import io.questdb.client.std.Numbers; -import io.questdb.client.std.str.StringSink; - -/** Schema-only floating formatter matching the server's Ryu presentation. */ -final class QwpSchemaDoubleFormatter { - private static final int EXP_SHIFT = 52; - private static final long EXP_BIT_MASK = 0x7ff0000000000000L; - private static final long SIGN_BIT_MASK = 0x8000000000000000L; - private static final long SIGNIFICAND_BIT_MASK = 0x000fffffffffffffL; - - private QwpSchemaDoubleFormatter() { - } - - static void append(StringSink sink, double value, int[] exponentScratch) { - long bits = Double.doubleToRawLongBits(value); - long mantissa = bits & SIGNIFICAND_BIT_MASK; - int exponent = (int) ((bits & EXP_BIT_MASK) >>> EXP_SHIFT); - if (exponent == 2047) { - if (mantissa != 0) { - sink.putAscii("NaN"); - } else if ((bits & SIGN_BIT_MASK) != 0) { - sink.putAscii("-Infinity"); - } else { - sink.putAscii("Infinity"); - } - return; - } - if ((bits & SIGN_BIT_MASK) != 0) { - sink.putAscii('-'); - } - if (exponent == 0 && mantissa == 0) { - sink.putAscii("0.0"); - return; - } - long output = QwpSchemaRyuDouble.d2d(mantissa, exponent, exponentScratch); - int length = QwpSchemaRyuDouble.decimalLength17(output); - int decimalExponent = exponentScratch[0] + length; - if (decimalExponent > 0 && decimalExponent < 8) { - appendFixedPositive(sink, output, length, decimalExponent); - } else if (decimalExponent <= 0 && decimalExponent > -3) { - appendFixedLeadingZero(sink, output, length, decimalExponent); - } else { - appendScientific(sink, output, length, decimalExponent); - } - } - - private static void appendFixedLeadingZero( - StringSink sink, long output, int length, int decimalExponent - ) { - sink.putAscii("0."); - for (int i = 0; i < -decimalExponent; i++) { - sink.putAscii('0'); - } - appendDigits(sink, output, length, 0, length); - } - - private static void appendFixedPositive( - StringSink sink, long output, int length, int decimalExponent - ) { - if (length <= decimalExponent) { - appendDigits(sink, output, length, 0, length); - for (int i = length; i < decimalExponent; i++) { - sink.putAscii('0'); - } - sink.putAscii(".0"); - } else { - appendDigits(sink, output, length, 0, decimalExponent); - sink.putAscii('.'); - appendDigits(sink, output, length, decimalExponent, length); - } - } - - private static void appendScientific( - StringSink sink, long output, int length, int decimalExponent - ) { - appendDigit(sink, output, length - 1); - sink.putAscii('.'); - if (length > 1) { - appendDigits(sink, output, length, 1, length); - } else { - sink.putAscii('0'); - } - sink.putAscii('E'); - Numbers.append(sink, decimalExponent - 1); - } - - private static void appendDigits(StringSink sink, long output, int length, int from, int to) { - for (int i = from; i < to; i++) { - appendDigit(sink, output, length - 1 - i); - } - } - - private static void appendDigit(StringSink sink, long output, int power) { - sink.putAscii((char) ('0' + (int) (output / Numbers.pow10[power] % 10))); - } -} diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index 5cccdda5b..6bda0ace7 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -70,7 +70,10 @@ public final class Numbers { private static final int[] insignificantDigitsNumber = new int[]{0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19}; private static final LongHexAppender[] longHexAppender = new LongHexAppender[Long.SIZE + 1]; private static final LongHexAppender[] longHexAppenderPad64 = new LongHexAppender[Long.SIZE + 1]; - private final static ThreadLocal tlDoubleDigitsBuffer = new ThreadLocal<>(() -> new char[21]); + // Scratch cell for RyuDouble.d2d()'s out-parameter. The server passes this + // through CharSink.ryuScratch(); the client's CharSink has no such method, + // so the single shared formatter keeps its own per-thread cell instead. + private final static ThreadLocal tlRyuScratch = new ThreadLocal<>(() -> new int[1]); private Numbers() { } @@ -175,47 +178,92 @@ public static void append(CharSink sink, double value) { } public static void append(CharSink sink, double value, int scale) { - final char[] digits = tlDoubleDigitsBuffer.get(); final long doubleBits = Double.doubleToRawLongBits(value); - boolean negative = (doubleBits & SIGN_BIT_MASK) != 0L; - long significantBitCount = doubleBits & SIGNIF_BIT_MASK; - int binExp = (int) ((doubleBits & EXP_BIT_MASK) >> EXP_SHIFT); - - if (binExp == 2047) { - if (significantBitCount == 0L) { - if (negative) { - sink.putAscii("-Infinity"); - } else { - sink.putAscii("Infinity"); + long ieeeMantissa = doubleBits & SIGNIF_BIT_MASK; + int ieeeExponent = (int) ((doubleBits & EXP_BIT_MASK) >> EXP_SHIFT); + + // NaN or Infinity + if (ieeeExponent == 2047) { + if (ieeeMantissa != 0) { + sink.putAscii("NaN"); + } else if ((doubleBits & SIGN_BIT_MASK) != 0) { + sink.putAscii("-Infinity"); + } else { + sink.putAscii("Infinity"); + } + return; + } + + // Negative sign (including -0.0) + if ((doubleBits & SIGN_BIT_MASK) != 0) { + sink.putAscii('-'); + } + + // Zero + if (ieeeExponent == 0 && ieeeMantissa == 0) { + sink.putAscii("0.0"); + return; + } + + // Decompose via Ryu + int[] e10 = tlRyuScratch.get(); + long output = RyuDouble.d2d(ieeeMantissa, ieeeExponent, e10); + int olength = RyuDouble.decimalLength17(output); + int decExp = e10[0] + olength; + + if (decExp > 0 && decExp < 8) { + // Fixed-point with integer part: e.g. "1234.567" or "1234000.0" + if (olength <= decExp) { + for (int i = 0; i < olength; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); + } + for (int i = olength; i < decExp; i++) { + sink.putAscii('0'); } + sink.putAscii(".0"); } else { - sink.putAscii("NaN"); + int fracDigits = olength - decExp; + if (scale < MAX_DOUBLE_SCALE && fracDigits > scale) { + fracDigits = Math.max(scale, 1); + } + for (int i = 0; i < decExp; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); + } + sink.putAscii('.'); + for (int i = 0; i < fracDigits; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - decExp - i] % 10))); + } + } + } else if (decExp <= 0 && decExp > -3) { + // Leading-zero fixed-point: e.g. "0.00123" + int leadingZeros = -decExp; + int totalFrac = leadingZeros + olength; + int digitsFromOutput = olength; + if (scale < MAX_DOUBLE_SCALE && totalFrac > scale) { + totalFrac = Math.max(scale, 1); + digitsFromOutput = Math.max(totalFrac - leadingZeros, 0); + } + int zerosToWrite = Math.min(leadingZeros, totalFrac); + sink.putAscii("0."); + for (int i = 0; i < zerosToWrite; i++) { + sink.putAscii('0'); + } + for (int i = 0; i < digitsFromOutput; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); } } else { - int fractionBits; - if (binExp == 0) { - if (significantBitCount == 0L) { - if (negative) { - sink.putAscii("-0.0"); - } else { - sink.putAscii("0.0"); - } - return; + // Scientific notation: e.g. "1.23E8" or "1.0E-4" + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1] % 10))); + sink.putAscii('.'); + if (olength > 1) { + for (int i = 1; i < olength; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); } - - int leadingZeros = Long.numberOfLeadingZeros(significantBitCount); - int shift = leadingZeros - (63 - EXP_SHIFT); - significantBitCount <<= shift; - binExp = 1 - shift; - fractionBits = 64 - leadingZeros; } else { - significantBitCount |= FRACT_HOB; - fractionBits = 53; + sink.putAscii('0'); } - - binExp -= EXP_BIAS; - - appendDouble0(binExp, significantBitCount, fractionBits, negative, digits, sink, scale); + sink.putAscii('E'); + append(sink, decExp - 1); } } @@ -480,347 +528,6 @@ public static long parseLong(Utf8Sequence sequence) throws NumericException { return parseLong0(sequence.asAsciiCharSequence(), sequence.size()); } - private static void appendDouble0( - int binExp, - long fractionBits, - int significantBitCount, - boolean negative, - char[] digits, - CharSink out, - int outScale - ) { - assert fractionBits > 0L; - assert (fractionBits & FRACT_HOB) != 0L; - - final int tailZeroes = Long.numberOfTrailingZeros(fractionBits); - final int fractBitCount = EXP_SHIFT + 1 - tailZeroes; - int decExp; - int firstDigitIndex; - int nDigits; - - final int tinyBitCount = Math.max(0, fractBitCount - binExp - 1); - if (binExp < MAX_SMALL_BIN_EXP + 1 && binExp > MIN_SMALL_BIN_EXP - 1 && tinyBitCount < LONG_5_POW.length && fractBitCount + N_5_BITS[tinyBitCount] < 64 && tinyBitCount == 0) { - int insignificant; - if (binExp > significantBitCount) { - insignificant = insignificantDigitsForPow2(binExp - significantBitCount - 1); - } else { - insignificant = 0; - } - - if (binExp >= EXP_SHIFT) { - fractionBits <<= binExp - EXP_SHIFT; - } else { - fractionBits >>>= EXP_SHIFT - binExp; - } - - // - int binExp2 = 0; - if (insignificant != 0) { - long pow10 = LONG_5_POW[insignificant] << insignificant; - long residue = fractionBits % pow10; - fractionBits /= pow10; - binExp2 += insignificant; - if (residue >= pow10 >> 1) { - ++fractionBits; - } - } - - int digitIndex = digits.length - 1; - int digit; - if (fractionBits <= Integer.MAX_VALUE) { - assert fractionBits > 0L : fractionBits; - - int fractRemaining = (int) fractionBits; - digit = fractRemaining % 10; - - for (fractRemaining /= 10; digit == 0; fractRemaining /= 10) { - ++binExp2; - digit = fractRemaining % 10; - } - - while (fractRemaining != 0) { - digits[digitIndex--] = (char) (digit + '0'); - ++binExp2; - digit = fractRemaining % 10; - fractRemaining /= 10; - } - - } else { - digit = (int) (fractionBits % 10L); - - for (fractionBits /= 10L; digit == 0; fractionBits /= 10L) { - ++binExp2; - digit = (int) (fractionBits % 10L); - } - - while (fractionBits != 0L) { - digits[digitIndex--] = (char) (digit + '0'); - ++binExp2; - digit = (int) (fractionBits % 10L); - fractionBits /= 10L; - } - - } - digits[digitIndex] = (char) (digit + '0'); - - decExp = binExp2 + 1; - firstDigitIndex = digitIndex; - nDigits = digits.length - digitIndex; - } else { - int estDecExp = estimateDecExpDouble(fractionBits, binExp); - int B5 = Math.max(0, -estDecExp); - int B2 = B5 + tinyBitCount + binExp; - int S5 = Math.max(0, estDecExp); - int S2 = S5 + tinyBitCount; - int M2 = B2 - significantBitCount; - fractionBits >>>= tailZeroes; - B2 -= fractBitCount - 1; - int common2factor = Math.min(B2, S2); - B2 -= common2factor; - S2 -= common2factor; - M2 -= common2factor; - if (fractBitCount == 1) { - --M2; - } - - if (M2 < 0) { - B2 -= M2; - S2 -= M2; - M2 = 0; - } - - int bBits = fractBitCount + B2 + (B5 < N_5_BITS.length ? N_5_BITS[B5] : B5 * 3); - int tenBits = S2 + 1 + (S5 + 1 < N_5_BITS.length ? N_5_BITS[S5 + 1] : (S5 + 1) * 3); - boolean low; - boolean high; - long lowDigitDifference; - int q; - int digitIndex; - if (bBits < 64 && tenBits < 64) { - if (bBits < 32 && tenBits < 32) { - int b = (int) fractionBits * SMALL_5_POW[B5] << B2; - int s = SMALL_5_POW[S5] << S2; - int m = SMALL_5_POW[B5] << M2; - int tens = s * 10; - digitIndex = 0; - q = b / s; - b = 10 * (b % s); - m *= 10; - low = b < m; - high = b + m > tens; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - for (; !low && !high; digits[digitIndex++] = (char) ('0' + q)) { - q = b / s; - b = 10 * (b % s); - m *= 10; - - assert q < 10 : q; - - if ((long) m > 0L) { - low = b < m; - high = b + m > tens; - } else { - low = true; - high = true; - } - } - - lowDigitDifference = ((long) b << 1) - tens; - } else { - long b = fractionBits * LONG_5_POW[B5] << B2; - long s = LONG_5_POW[S5] << S2; - long m = LONG_5_POW[B5] << M2; - long tens = s * 10L; - digitIndex = 0; - q = (int) (b / s); - b = 10L * (b % s); - m *= 10L; - low = b < m; - high = b + m > tens; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - for (; !low && !high; digits[digitIndex++] = (char) ('0' + q)) { - q = (int) (b / s); - b = 10L * (b % s); - m *= 10L; - - assert q < 10 : q; - - if (m > 0L) { - low = b < m; - high = b + m > tens; - } else { - low = true; - high = true; - } - } - lowDigitDifference = (b << 1) - tens; - } - } else { - FdBig sVal = FdBig.valueOfPow52(S5, S2); - final int shiftBias = sVal.getNormalizationBias(); - sVal = sVal.leftShift(shiftBias); - FdBig bVal = FdBig.valueOfMulPow52(fractionBits, B5, B2 + shiftBias); - FdBig mVal = FdBig.valueOfPow52(B5 + 1, M2 + shiftBias + 1); - FdBig tensVal = FdBig.valueOfPow52(S5 + 1, S2 + shiftBias + 1); - digitIndex = 0; - q = bVal.quoRemIteration(sVal); - low = bVal.cmp(mVal) < 0; - high = tensVal.addAndCmp(bVal, mVal) <= 0; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - while (!low && !high) { - q = bVal.quoRemIteration(sVal); - - assert q < 10 : q; - - mVal = mVal.multBy10(); - low = bVal.cmp(mVal) < 0; - high = tensVal.addAndCmp(bVal, mVal) <= 0; - digits[digitIndex++] = (char) ('0' + q); - } - - if (high && low) { - bVal = bVal.leftShift(1); - lowDigitDifference = bVal.cmp(tensVal); - } else { - lowDigitDifference = 0L; - } - } - - decExp = estDecExp + 1; - firstDigitIndex = 0; - nDigits = digitIndex; - if (high) { - if (low) { - if (lowDigitDifference == 0L) { - if ((digits[firstDigitIndex + nDigits - 1] & 1) != 0) { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } else if (lowDigitDifference > 0L) { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } else { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } - } - - appendDouble00(digits, firstDigitIndex, nDigits, negative, decExp, out, outScale); - } - - private static void appendDouble00( - char[] digits, - int firstDigitIndex, - int nDigits, - boolean isNegative, - int decExp, - CharSink sink, - int outScale - ) { - assert nDigits <= MAX_DOUBLE_SCALE : nDigits; - if (isNegative) { - sink.putAscii('-'); - } - - int exp; - if (decExp > 0 && decExp < 8) { - exp = Math.min(nDigits, decExp); - sink.putAscii(digits, firstDigitIndex, exp); - if (exp < decExp) { - exp = decExp - exp; - sink.fillAscii('0', exp); - sink.putAscii('.'); - sink.putAscii('0'); - } else { - sink.putAscii('.'); - if (exp < nDigits) { - sink.putAscii(digits, firstDigitIndex + exp, Math.min(nDigits - exp, outScale)); - } else { - sink.putAscii('0'); - } - } - } else if (decExp <= 0 && decExp > -3) { - sink.putAscii('0').putAscii('.'); - if (decExp != 0) { - sink.fillAscii('0', -decExp); - } - - sink.putAscii(digits, firstDigitIndex, Math.min(nDigits, outScale)); - } else { - sink.putAscii(digits[firstDigitIndex]); - sink.putAscii('.'); - if (nDigits > 1) { - sink.putAscii(digits, firstDigitIndex + 1, nDigits - 1); - } else { - sink.putAscii('0'); - } - - sink.putAscii('E'); - if (decExp <= 0) { - sink.putAscii('-'); - exp = -decExp + 1; - } else { - exp = decExp - 1; - } - - if (exp < 10) { - sink.putAscii((char) (exp + '0')); - } else if (exp < 100) { - sink.putAscii((char) (exp / 10 + '0')); - sink.putAscii((char) (exp % 10 + '0')); - } else { - sink.putAscii((char) (exp / 100 + '0')); - exp %= 100; - sink.putAscii((char) (exp / 10 + '0')); - sink.putAscii((char) (exp % 10 + '0')); - } - } - } - private static void appendInt10(CharSink sink, int i) { int c; sink.putAscii((char) ('0' + i / 1000000000)); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java b/core/src/main/java/io/questdb/client/std/RyuDouble.java similarity index 98% rename from core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java rename to core/src/main/java/io/questdb/client/std/RyuDouble.java index edba00986..e20fb56f7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaRyuDouble.java +++ b/core/src/main/java/io/questdb/client/std/RyuDouble.java @@ -22,22 +22,24 @@ * ******************************************************************************/ -package io.questdb.client.cutlass.qwp.protocol; +package io.questdb.client.std; /** - * Schema-only port of the QuestDB server's {@code io.questdb.std.RyuDouble} - * at revision {@code 12a33d651e51e2682e7a448c8db5168fc72dfad3}. + * Port of the QuestDB server's {@code io.questdb.std.RyuDouble}, kept + * byte-compatible with it so that client-side and server-side formatting of + * the same double produce identical text. * Ryu algorithm for converting IEEE 754 double-precision floating-point numbers * to their shortest decimal representation. *

      * Based on the paper "Ryu: Fast Float-to-String Conversion" by Ulf Adams (2018). - * This implementation uses 128-bit arithmetic via {@code Math.multiplyHigh} (Java 9+). + * This implementation uses 128-bit arithmetic via {@link Compat#multiplyHigh}, + * which is {@code Math.multiplyHigh} on Java 9+ and a hand-rolled equivalent on Java 8. *

      * The core method {@link #d2d} returns the shortest decimal significand as a {@code long} * (1-17 digits) and writes the decimal exponent to a caller-provided array. The result * satisfies: {@code value = output * 10^e10}. */ -class QwpSchemaRyuDouble { +class RyuDouble { private static final int DOUBLE_POW5_BITCOUNT = 125; private static final int DOUBLE_POW5_INV_BITCOUNT = 125; // ceil(2^(pow5bits(q) - 1 + 125) / 5^q) for q in [0, 291], stored as (hi64, lo64) pairs. @@ -373,11 +375,11 @@ private static long[] initPow5Split() { */ // Requires shift in [65, 127] (guaranteed by the Ryu algorithm). private static long mulShift64(long m, long mul0, long mul1, int shift) { - // Math.multiplyHigh is signed; correct for unsigned by adding m + // Compat.multiplyHigh is signed; correct for unsigned by adding m // when the other operand is negative (MSB set). - long high1 = Math.multiplyHigh(m, mul1) + ((mul1 >> 63) & m); + long high1 = Compat.multiplyHigh(m, mul1) + ((mul1 >> 63) & m); long low0 = m * mul0; - long high0 = Math.multiplyHigh(m, mul0) + ((mul0 >> 63) & m); + long high0 = Compat.multiplyHigh(m, mul0) + ((mul0 >> 63) & m); long mid = low0 + high1; long carry = Long.compareUnsigned(mid, low0) < 0 ? 1L : 0L; diff --git a/core/src/main/java11/io/questdb/client/std/Compat.java b/core/src/main/java11/io/questdb/client/std/Compat.java index d0c6b1a66..f6d39aba4 100644 --- a/core/src/main/java11/io/questdb/client/std/Compat.java +++ b/core/src/main/java11/io/questdb/client/std/Compat.java @@ -24,8 +24,6 @@ package io.questdb.client.std; -import java.lang.reflect.Method; - /** * JDK-version-specific helpers. This is the Java 9+ variant; the parallel copy * under {@code src/main/java8} provides Java 8 implementations of the same API. @@ -48,27 +46,19 @@ public static long currentPid() { } /** - * Spin-loop pause hint. Delegates to {@code Thread.onSpinWait()} on Java 9+; - * the Java 8 variant is a no-op (no such hint exists there). + * Returns the most significant 64 bits of the 128-bit product of two longs. + * Delegates to {@code Math.multiplyHigh()} on Java 9+, which HotSpot + * intrinsifies; the Java 8 variant computes the same result by hand. */ - public static void onSpinWait() { - Thread.onSpinWait(); + public static long multiplyHigh(long x, long y) { + return Math.multiplyHigh(x, y); } /** - * Opens {@code java.base/jdk.internal.math} to this module so that - * {@code FDBigInteger} is reachable at runtime, mirroring the - * {@code --add-exports} flag used at compile time. + * Spin-loop pause hint. Delegates to {@code Thread.onSpinWait()} on Java 9+; + * the Java 8 variant is a no-op (no such hint exists there). */ - static void exportFdBigInteger() { - try { - Module base = System.class.getModule(); - Module current = Compat.class.getModule(); - Method implAddExports = Module.class.getDeclaredMethod("implAddExports", String.class, Module.class); - Unsafe.makeAccessible(implAddExports); - implAddExports.invoke(base, "jdk.internal.math", current); - } catch (ReflectiveOperationException e) { - e.printStackTrace(System.out); - } + public static void onSpinWait() { + Thread.onSpinWait(); } } diff --git a/core/src/main/java11/io/questdb/client/std/FdBig.java b/core/src/main/java11/io/questdb/client/std/FdBig.java deleted file mode 100644 index 0f019ab28..000000000 --- a/core/src/main/java11/io/questdb/client/std/FdBig.java +++ /dev/null @@ -1,82 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import jdk.internal.math.FDBigInteger; - -/** - * Thin bridge over the JDK's internal {@code FDBigInteger}, used only by the - * {@link Numbers} double-to-string slow path. The bignum class lives in - * {@code jdk.internal.math} on Java 9+; the parallel copy of this file under - * {@code src/main/java8} targets {@code sun.misc.FDBigInteger}. Keeping the - * type behind this wrapper lets {@code Numbers} carry a single, JDK-agnostic - * copy of the algorithm. - */ -final class FdBig { - static { - // Mirror the compile-time --add-exports: grant this module runtime - // access to jdk.internal.math before any FDBigInteger reference is - // resolved. No-op on Java 8 (sun.misc is open). - Compat.exportFdBigInteger(); - } - - private final FDBigInteger value; - - private FdBig(FDBigInteger value) { - this.value = value; - } - - static FdBig valueOfMulPow52(long value, int p5, int p2) { - return new FdBig(FDBigInteger.valueOfMulPow52(value, p5, p2)); - } - - static FdBig valueOfPow52(int p5, int p2) { - return new FdBig(FDBigInteger.valueOfPow52(p5, p2)); - } - - int addAndCmp(FdBig x, FdBig y) { - return value.addAndCmp(x.value, y.value); - } - - int cmp(FdBig other) { - return value.cmp(other.value); - } - - int getNormalizationBias() { - return value.getNormalizationBias(); - } - - FdBig leftShift(int shift) { - return new FdBig(value.leftShift(shift)); - } - - FdBig multBy10() { - return new FdBig(value.multBy10()); - } - - int quoRemIteration(FdBig s) { - return value.quoRemIteration(s.value); - } -} diff --git a/core/src/main/java8/io/questdb/client/std/Compat.java b/core/src/main/java8/io/questdb/client/std/Compat.java index 3d9073e79..1a9459e28 100644 --- a/core/src/main/java8/io/questdb/client/std/Compat.java +++ b/core/src/main/java8/io/questdb/client/std/Compat.java @@ -59,13 +59,25 @@ public static long currentPid() { * Spin-loop pause hint. No-op on Java 8 — {@code Thread.onSpinWait()} does * not exist there. The parallel Java 9+ variant delegates to it. */ - public static void onSpinWait() { - } - /** - * No-op on Java 8: {@code sun.misc} is open, so {@code FDBigInteger} needs - * no module export. Present for symmetry with the Java 9+ variant. + * Returns the most significant 64 bits of the 128-bit product of two longs. + * Java 8 has no {@code Math.multiplyHigh()}, so this splits both operands + * into 32-bit halves and reassembles the high word from the partial + * products. The Java 9+ variant delegates to the intrinsic instead. */ - static void exportFdBigInteger() { + public static long multiplyHigh(long x, long y) { + long x1 = x >> 32; + long x2 = x & 0xFFFFFFFFL; + long y1 = y >> 32; + long y2 = y & 0xFFFFFFFFL; + long z2 = x2 * y2; + long t = x1 * y2 + (z2 >>> 32); + long z1 = t & 0xFFFFFFFFL; + long z0 = t >> 32; + z1 += x2 * y1; + return x1 * y1 + z0 + (z1 >> 32); + } + + public static void onSpinWait() { } } diff --git a/core/src/main/java8/io/questdb/client/std/FdBig.java b/core/src/main/java8/io/questdb/client/std/FdBig.java deleted file mode 100644 index 7ca6600ef..000000000 --- a/core/src/main/java8/io/questdb/client/std/FdBig.java +++ /dev/null @@ -1,81 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import sun.misc.FDBigInteger; - -/** - * Thin bridge over the JDK's internal {@code FDBigInteger}, used only by the - * {@link Numbers} double-to-string slow path. This is the Java 8 variant: the - * bignum class lives in {@code sun.misc}. The parallel copy under - * {@code src/main/java11} targets {@code jdk.internal.math.FDBigInteger}. The - * public surface is identical across both JDKs, so {@code Numbers} carries a - * single, JDK-agnostic copy of the algorithm. - */ -final class FdBig { - static { - // No-op on Java 8: sun.misc is open, no module export is needed. Kept - // for symmetry with the Java 9+ variant. - Compat.exportFdBigInteger(); - } - - private final FDBigInteger value; - - private FdBig(FDBigInteger value) { - this.value = value; - } - - static FdBig valueOfMulPow52(long value, int p5, int p2) { - return new FdBig(FDBigInteger.valueOfMulPow52(value, p5, p2)); - } - - static FdBig valueOfPow52(int p5, int p2) { - return new FdBig(FDBigInteger.valueOfPow52(p5, p2)); - } - - int addAndCmp(FdBig x, FdBig y) { - return value.addAndCmp(x.value, y.value); - } - - int cmp(FdBig other) { - return value.cmp(other.value); - } - - int getNormalizationBias() { - return value.getNormalizationBias(); - } - - FdBig leftShift(int shift) { - return new FdBig(value.leftShift(shift)); - } - - FdBig multBy10() { - return new FdBig(value.multBy10()); - } - - int quoRemIteration(FdBig s) { - return value.quoRemIteration(s.value); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java index f49bde362..1217a2d13 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java @@ -180,7 +180,9 @@ public void testOneMiBBoundAppliesOnlyToUpdates() { byte[] payload = missingPayload(); byte[] update = java.util.Arrays.copyOf(base, base.length + 2 + 2 + 1 + 4 + payload.length); ByteBuffer trailer = ByteBuffer.wrap(update).order(ByteOrder.LITTLE_ENDIAN); - trailer.position(base.length).putShort((short) 1).putShort((short) 1).put((byte) 't').putInt(payload.length).put(payload); + // Not chained off position(): that returns Buffer, not ByteBuffer, on Java 8. + trailer.position(base.length); + trailer.putShort((short) 1).putShort((short) 1).put((byte) 't').putInt(payload.length).put(payload); update[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES); withNative(update, (ptr, n) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, n, true))); } diff --git a/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java b/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java index 17ca50d58..0281c173b 100644 --- a/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java +++ b/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java @@ -29,7 +29,7 @@ /** * Standalone (no JUnit) smoke check that double formatting works against the * packaged jar on the running JDK. The extreme-exponent values exercise the - * FdBig fallback of {@code Numbers.appendDouble0}, which resolves + * Ryu double formatter, which resolves * {@code FDBigInteger} from a different JDK-internal package on Java 8 vs 9+; * a jar packaged with the wrong bridge dies here with * {@code NoClassDefFoundError: sun/misc/FDBigInteger} (the 1.3.5-1.3.7 @@ -42,7 +42,7 @@ public static void main(String[] args) { double[] values = { 0.0d, 123.456d, - // FdBig slow-path values + // formerly FdBig slow-path values 1.0E-300, Double.MIN_VALUE, Double.MAX_VALUE, diff --git a/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java b/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java index 549768a71..854b0e783 100644 --- a/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java +++ b/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java @@ -43,14 +43,15 @@ *

    6. the module name silently changing from {@code io.questdb.client} to the * filename-derived {@code questdb.client} (no module-info.class on JDK 8 * builds and no Automatic-Module-Name),
      7. - *
    7. JDK 8 builds shipping only the {@code sun.misc.FDBigInteger} FdBig - * bridge, which throws {@code NoClassDefFoundError} on Java 9+ without the - * {@code META-INF/versions/11} counterpart.
      8. + *
    8. JDK 8 builds shipping only the Java 8 {@code Compat} shim, whose + * {@code multiplyHigh} fallback the Ryu double formatter calls on every + * value, without the {@code META-INF/versions/11} counterpart that lets + * Java 9+ use the {@code Math.multiplyHigh} intrinsic instead.
      9. * */ public class JarPackagingIT { - private static final String FD_BIG_ENTRY = "io/questdb/client/std/FdBig.class"; - private static final String VERSIONED_FD_BIG_ENTRY = "META-INF/versions/11/" + FD_BIG_ENTRY; + private static final String COMPAT_ENTRY = "io/questdb/client/std/Compat.class"; + private static final String VERSIONED_COMPAT_ENTRY = "META-INF/versions/11/" + COMPAT_ENTRY; @Test public void testDoubleFormattingAgainstPackagedJar() throws Exception { @@ -93,16 +94,16 @@ public void testJarLayout() throws Exception { // JDK 8 build: the shipping layout. Root classes target Java 8, // the java11 bridge rides in META-INF/versions/11. Assert.assertTrue( - "root FdBig of a JDK 8 build must use sun.misc.FDBigInteger", - classReferences(jar, FD_BIG_ENTRY, "sun/misc/FDBigInteger") - ); - Assert.assertTrue( - "META-INF/versions/11 FdBig must use jdk.internal.math.FDBigInteger", - classReferences(jar, VERSIONED_FD_BIG_ENTRY, "jdk/internal/math/FDBigInteger") + "root Compat of a JDK 8 build must use the Java 8 ManagementFactory variant", + classReferences(jar, COMPAT_ENTRY, "java/lang/management/ManagementFactory") ); Assert.assertNotNull( "META-INF/versions/11 must carry the java11 Compat shim", - jar.getEntry("META-INF/versions/11/io/questdb/client/std/Compat.class") + jar.getEntry(VERSIONED_COMPAT_ENTRY) + ); + Assert.assertTrue( + "META-INF/versions/11 Compat must use the Math.multiplyHigh intrinsic", + classReferences(jar, VERSIONED_COMPAT_ENTRY, "multiplyHigh") ); // every source in src/main/java11 must ship in versions/11 -- a file added // to the source root but missed by the packaging step would recreate the @@ -117,8 +118,8 @@ public void testJarLayout() throws Exception { // JDK 11+ build: dev/smoke only, never shipped. Root classes are the // java11 variants and the real module descriptor is present. Assert.assertTrue( - "root FdBig of a JDK 11+ build must use jdk.internal.math.FDBigInteger", - classReferences(jar, FD_BIG_ENTRY, "jdk/internal/math/FDBigInteger") + "root Compat of a JDK 11+ build must use the Math.multiplyHigh intrinsic", + classReferences(jar, COMPAT_ENTRY, "multiplyHigh") ); Assert.assertNotNull("module-info.class missing", jar.getEntry("module-info.class")); } diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index 38dd233b0..5b5c6c15b 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -140,7 +140,10 @@ public void testFormatDoubleExpNeg() { public void testFormatDoubleFast() { sink.clear(); Numbers.append(sink, -5.9522650387500933e18); - TestUtils.assertEquals("-5.9522650387500933E18", sink); + // Shortest form that still reads back to the same double. The previous + // formatter emitted a 17th digit ("...0933E18"); both parse to + // -0x1.4a6ac610aeb44p62, and Double.toString agrees with this one. + TestUtils.assertEquals("-5.952265038750093E18", sink); } @Test From eeb0bbd729b71ca85318e546a38bbac008afa37d Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 17 Sep 2026 15:12:45 +0200 Subject: [PATCH 20/51] Share one row-failure policy across QWP setters Every row-building setter in QwpWebSocketSender repeated the same two catch blocks: roll the partial row back, run the one-refresh rule for a LineSenderSchemaException, and rethrow everything else. The policy lived in 31 copies, and D045 already recorded one setter (CHAR) that shipped without the schema branch. onRowFailure() now owns that policy. It rolls the whole row back first, so a failed refresh cannot leave a half-written row, then hands a schema rejection to refreshAfterSchemaRejection() and lets every other failure propagate unchanged. All 36 row guards call it from a single catch. The two catches in installSchemaBinding() guard buffer cleanup, not rows, and stay as they are. Behaviour change: the four longArray() overloads had only the generic catch, so their UNSUPPORTED_FEATURE rejection skipped the refresh. They now follow the standard path: the first rejection costs one DESCRIBE round trip and reports SCHEMA_CHANGED when the target column changed. atNow() also routes through the helper, but its null column name keeps it on the no-refresh path. QwpSchemaSenderIntegrationTest covers both longArray() outcomes, asserts whole-row rollback and exactly one refresh, and extends the null no-op test with the four longArray() null forms. Co-Authored-By: Claude Fable 5.1 --- .../qwp/client/QwpWebSocketSender.java | 222 ++++-------------- .../QwpSchemaSenderIntegrationTest.java | 61 +++++ 2 files changed, 113 insertions(+), 170 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3c9e46f01..ccefbfbf2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1040,12 +1040,8 @@ public void at(long timestamp, ChronoUnit unit) { long micros = toMicros(timestamp, unit); atMicros(micros); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, null, true); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, null, true); } } @@ -1062,12 +1058,8 @@ public void at(Instant timestamp) { } long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; atMicros(micros); - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, null, true); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, null, true); } } @@ -1080,8 +1072,7 @@ public void atNow() { // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, null); } } @@ -1161,12 +1152,8 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { if (col != null) { col.addBinary(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -1192,12 +1179,8 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice "BINARY slice cannot be null; mark the row null via the null bitmap instead"); } return binaryColumn(columnName, slice.ptr(), slice.size()); - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } } @@ -1220,12 +1203,8 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l if (col != null) { col.addBinary(ptr, len); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -1244,12 +1223,8 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { if (col != null) { col.addBoolean(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -1279,12 +1254,8 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { if (col != null) { col.addByte(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -1320,12 +1291,8 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { if (col != null) { col.addShort((short) value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -1668,12 +1635,8 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (col != null) { col.addDecimal64(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1693,12 +1656,8 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (col != null) { col.addDecimal128(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1718,12 +1677,8 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (col != null) { col.addDecimal256(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1744,12 +1699,8 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { if (col != null) { col.addDecimal256(currentDecimal256); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1769,12 +1720,8 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (col != null) { col.addDoubleArray(values); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1794,12 +1741,8 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (col != null) { col.addDoubleArray(values); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1819,12 +1762,8 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (col != null) { col.addDoubleArray(values); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1844,12 +1783,8 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (col != null) { col.addDoubleArray(array); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, name); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -1868,12 +1803,8 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { if (col != null) { col.addDouble(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -1898,12 +1829,8 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { if (col != null) { col.addFloat(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -2052,12 +1979,8 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int if (col != null) { col.addGeoHash(maskGeoHashBits(bits, precisionBits), precisionBits); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -2086,12 +2009,8 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va binding.geoHashColumn(columnName, value); return this; } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } if (value == null) { throw new LineSenderException( @@ -2482,12 +2401,8 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { if (col != null) { col.addInt(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -2519,12 +2434,8 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { if (col != null) { col.addIPv4(address); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -2569,12 +2480,8 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre binding.ipv4Column(columnName, address); return this; } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { throw new LineSenderException( @@ -2621,12 +2528,8 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l if (col != null) { col.addLong256(l0, l1, l2, l3); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -2647,8 +2550,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -2669,8 +2571,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -2691,8 +2592,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -2713,8 +2613,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { col.addLongArray(array); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, name); } return this; } @@ -2733,12 +2632,8 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { if (col != null) { col.addLong(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -3054,12 +2949,8 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { if (col != null) { col.addShort(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -3178,12 +3069,8 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val if (col != null) { col.addString(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -3202,12 +3089,8 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { if (col != null) { col.addSymbol(value); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -3267,12 +3150,8 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C col.addLong(micros); } } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -3292,12 +3171,8 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value if (col != null) { col.addLong(micros); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -3323,12 +3198,8 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) if (col != null) { col.addUuid(hi, lo); } - } catch (LineSenderSchemaException e) { - rollbackRow(); - throw refreshAfterSchemaRejection(e, columnName); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e, columnName); } return this; } @@ -4575,11 +4446,22 @@ private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest, QwpSchem return replacementBinding; } - private RuntimeException refreshAfterSchemaRejection( - LineSenderSchemaException rejection, - CharSequence columnName - ) { - return refreshAfterSchemaRejection(rejection, columnName, false); + // Single home of the row-failure policy: roll the whole row back first, so a + // failed refresh cannot leave a half-written row, then apply the one-refresh + // rule to a schema rejection. Every other failure propagates unchanged. + private RuntimeException onRowFailure(Throwable e, CharSequence columnName) { + return onRowFailure(e, columnName, false); + } + + private RuntimeException onRowFailure(Throwable e, CharSequence columnName, boolean isDesignatedTimestamp) { + rollbackRow(); + if (e instanceof LineSenderSchemaException) { + return refreshAfterSchemaRejection((LineSenderSchemaException) e, columnName, isDesignatedTimestamp); + } + if (e instanceof Error) { + throw (Error) e; + } + return (RuntimeException) e; } private RuntimeException refreshAfterSchemaRejection( diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 90be992f7..e420ce003 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -10,6 +10,7 @@ import io.questdb.client.SenderConnectionEvent; import io.questdb.client.cairo.ColumnType; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; @@ -975,6 +976,62 @@ public void testDoubleArrayGenerationRemainsPinnedAcrossRankChange() throws Exce }); } + @Test + public void testLongArrayRejectionRollsBackAndRefreshesOnce() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1221, 1231, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.longArray("value", new long[]{99L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + // The frame carries the two DOUBLE[] rows only: the partial row that + // held failed_b rolled back, so no extra column reaches the wire. + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events", 1221, 1231); + Assert.assertEquals("setter must use the standard one-refresh path", 2, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongArrayRejectionSurfacesSchemaChangeAfterRebind() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1241, 1251, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + handler.targetType = array1d; + handler.version = 1252; + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.longArray("value", new long[]{3L, 4L}); + Assert.fail("expected target-changing schema refresh"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + } + sender.doubleArray("value", new double[]{3.0, 4.0}).atNow(); + sender.flush(); + // The rebound block holds the recovered row alone: rollback dropped + // failed_b before the refresh installed the new generation. + new FrameReader(handler.awaitDataFrame()) + .twoDoubleArrayRankBlocks("events", 1241, 1251, 1252); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + @Test public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { assertMemoryLeak(() -> { @@ -1598,6 +1655,10 @@ public void testNullNoopDoesNotTriggerNegotiation() throws Exception { try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").doubleArray("values", (double[]) null); + sender.longArray("values", (long[]) null); + sender.longArray("values", (long[][]) null); + sender.longArray("values", (long[][][]) null); + sender.longArray("values", (LongArray) null); sender.ipv4Column("address", (CharSequence) null); Assert.assertEquals(0, handler.describeRequests.get()); sender.cancelRow(); From 4275ad7a286b4edd23c86eaa655c97cbde7498ac Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 17 Sep 2026 15:36:04 +0200 Subject: [PATCH 21/51] Skip schema refresh for unsupported long arrays The QWP longArray setters cannot become valid after a schema change. Preserve their UNSUPPORTED_FEATURE rejection without issuing DESCRIBE, while retaining the shared row rollback helper. Cover all four overloads, rollback, changed schemas and unavailable metadata. Keep double-array schema recovery and null no-ops unchanged. --- .../qwp/client/QwpWebSocketSender.java | 9 +- .../QwpSchemaSenderIntegrationTest.java | 93 ++++++++++++++----- 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index ccefbfbf2..07d2b797b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2550,7 +2550,8 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + // LONG_ARRAY is unimplemented; a schema refresh cannot make it valid. + throw onRowFailure(e, null); } return this; } @@ -2571,7 +2572,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e, null); } return this; } @@ -2592,7 +2593,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e, null); } return this; } @@ -2613,7 +2614,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { col.addLongArray(array); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e, null); } return this; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index e420ce003..339393e68 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -977,35 +977,51 @@ public void testDoubleArrayGenerationRemainsPinnedAcrossRankChange() throws Exce } @Test - public void testLongArrayRejectionRollsBackAndRefreshesOnce() throws Exception { + public void testLongArrayRejectionRollsBackWithoutRefresh() throws Exception { assertMemoryLeak(() -> { - int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); - LongTextSchemaHandler handler = new LongTextSchemaHandler(1221, 1231, array2d); - try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { - sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + for (int overload = 0; overload < 4; overload++) { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1221, 1231, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); - sender.table("events").uuidColumn("failed_b", 11, 12); - try { - sender.longArray("value", new long[]{99L}); - Assert.fail("expected unsupported LONG_ARRAY setter"); - } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); - } + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + switch (overload) { + case 0: + sender.longArray("value", new long[]{99L}); + break; + case 1: + sender.longArray("value", new long[][]{{99L}}); + break; + case 2: + sender.longArray("value", new long[][][]{{{99L}}}); + break; + default: + try (LongArray array = new LongArray(1)) { + sender.longArray("value", array.append(99L)); + } + break; + } + Assert.fail("expected unsupported LONG_ARRAY setter [overload=" + overload + "]"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } - sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); - sender.flush(); - // The frame carries the two DOUBLE[] rows only: the partial row that - // held failed_b rolled back, so no extra column reaches the wire. - new FrameReader(handler.awaitDataFrame()) - .doubleArrayTable("events", 1221, 1231); - Assert.assertEquals("setter must use the standard one-refresh path", 2, - handler.describeRequests.get()); + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + // Only the two completed DOUBLE[] rows reach the wire; failed_b is rolled back. + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events", 1221, 1231); + Assert.assertEquals("unsupported input must not refresh [overload=" + overload + "]", 1, + handler.describeRequests.get()); + } } }); } @Test - public void testLongArrayRejectionSurfacesSchemaChangeAfterRebind() throws Exception { + public void testLongArrayRejectionDoesNotRefreshAfterSchemaChange() throws Exception { assertMemoryLeak(() -> { int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); @@ -1014,9 +1030,20 @@ public void testLongArrayRejectionSurfacesSchemaChangeAfterRebind() throws Excep sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); handler.targetType = array1d; handler.version = 1252; - sender.table("events").uuidColumn("failed_b", 11, 12); + for (int attempt = 0; attempt < 2; attempt++) { + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.longArray("value", new long[]{3L, 4L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("schema changes cannot implement LONG_ARRAY", 1, + handler.describeRequests.get()); + } + // A supported input still discovers the rank change and can be retried. try { - sender.longArray("value", new long[]{3L, 4L}); + sender.doubleArray("value", new double[]{3.0, 4.0}); Assert.fail("expected target-changing schema refresh"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); @@ -1032,6 +1059,26 @@ public void testLongArrayRejectionSurfacesSchemaChangeAfterRebind() throws Excep }); } + @Test(timeout = 10_000) + public void testLongArrayRejectionDoesNotDependOnDescribeAvailability() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.respondToDescribe = false; + ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(100); + sender.table("events").stringColumn("partial", "discard"); + try { + sender.longArray("id", new long[]{99L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + @Test public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { assertMemoryLeak(() -> { From 28328330c72677c13b5c03867afbd43f7fee4ee1 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 17 Sep 2026 16:28:12 +0200 Subject: [PATCH 22/51] Adopt schema mode at the batch boundary A QWP frame has one wire family, so a flush that held both legacy and schema table blocks needed its own delivery pipeline. flushPendingRowsMixed() duplicated the split path's pre-flight sizing, dictionary-chunk fallback, publish loop, baseline advance and commit bookkeeping, and it encoded every block two to three times. Row-boundary adoption was the only source of such batches. A batch now has one wire contract, selected by its first committed row. bindingForEffectiveWrite() keeps returning the legacy contract while a legacy batch is pending, and the first row after the flush adopts schema mode. The upgrade stays one-way and is delayed by at most one batch. This deletes flushPendingRowsMixed(), mixedFramesFit(), encodeSingleTableFrame() and isMixedSchemaBatch(); the encoder is unchanged. Trade-off: rows written after the supporting handshake but inside the pending legacy batch keep legacy conversion, for example microsecond Instant truncation. Which rows preceded the handshake was already a timing race. The change also fixes an existing defect. A legacy row, a flush, a supporting reconnect and then a row on the same table threw IllegalStateException from the column setter: installSchemaBinding() bound in place whenever the buffer had no rows, but a flushed legacy buffer keeps its column layout. It now binds in place only a buffer with no columns and replaces the buffer otherwise. The two upgrade integration tests now assert the batch-boundary rule and cover the flushed-legacy-buffer path. The design document and the journal record the superseded row-boundary rule. Co-Authored-By: Claude Fable 5.1 --- .../qwp/client/QwpWebSocketSender.java | 124 +++--------------- .../QwpSchemaSenderIntegrationTest.java | 27 ++-- design/schema-aware-sender-journal.md | 24 ++++ design/schema-aware-sender.md | 35 ++--- 4 files changed, 82 insertions(+), 128 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 07d2b797b..6c5820caf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -334,6 +334,9 @@ public class QwpWebSocketSender implements Sender { // (Invariant B); endpoint-policy and transport failures stay // contained in that loop and never reach the producer. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; + // Wire contract of the pending batch, selected by its first committed row. + // Meaningful only while pendingRowCount > 0. Producer-thread only. + private boolean isPendingBatchLegacy; private boolean ownsCursorEngine; // Whether close() may let the engine reclaim the parent-anchored LOGICAL slot lock. // False only while an outer frame holds it: Sender.build() acquires it for the whole @@ -4387,8 +4390,14 @@ private QwpSchemaBinding bindingForEffectiveWrite() { return pinned; } ensureConnected(); - if (initialConnectMode == Sender.InitialConnectMode.ASYNC - && !cursorEngine.requiresSchema()) { + // A batch has one wire contract, because a QWP frame has one wire family. + // While the pending batch holds legacy rows, later rows stay legacy even if + // a supporting handshake has landed since; the first row after the flush + // adopts schema mode. The upgrade is one-way, so a schema batch never + // reverts. + if ((pendingRowCount > 0 && isPendingBatchLegacy) + || (initialConnectMode == Sender.InitialConnectMode.ASYNC + && !cursorEngine.requiresSchema())) { schemaResolutionFresh[0] = false; return null; } @@ -4407,7 +4416,11 @@ private QwpSchemaBinding bindingForEffectiveWrite() { } private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest, QwpSchemaBinding pinned) { - if (pinned == null && currentTableBuffer.getRowCount() == 0) { + // Bind in place only a pristine buffer. A flushed legacy buffer has no rows + // but keeps its legacy column layout, which a schema binding cannot adopt; + // the replacement below covers it. + if (pinned == null && currentTableBuffer.getRowCount() == 0 + && currentTableBuffer.getColumnCount() == 0) { return new QwpSchemaBinding(currentTableBuffer, latest); } QwpTableBuffer replacement = new QwpTableBuffer(currentTableName, this); @@ -4571,10 +4584,6 @@ private void flushPendingRows(boolean deferCommit) { // up the new cap. int cap = serverMaxBatchSize; int deltaBaseline = symbolDeltaBaseline(); - if (isMixedSchemaBatch(tableCount)) { - flushPendingRowsMixed(deferCommit, cap, deltaBaseline); - return; - } int combinedBodyStart = encodeCombinedFrame(tableCount, deferCommit, deltaBaseline); int messageSize = encoder.finishMessage(); @@ -4657,104 +4666,6 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { } } - /** - * Flushes the rare live legacy-to-schema upgrade batch. Each block is its - * own frame because a QWP frame has one wire family. The first pass sizes - * every frame before anything is published; the second pass re-encodes and - * publishes in collection order, with one commit boundary for the batch. - */ - private void flushPendingRowsMixed(boolean deferCommit, int cap, int deltaBaseline) { - final int blockCount = flushTableBuffers.size(); - int sizingBaseline = deltaBaseline; - boolean fits = mixedFramesFit(cap, sizingBaseline); - boolean dictionaryChunksAwaitCommit = false; - if (!fits && !deltaDictEnabled && currentBatchMaxSymbolId > deltaBaseline - && mixedFramesFit(cap, currentBatchMaxSymbolId)) { - publishDictionaryChunks(cap, deltaBaseline + 1, currentBatchMaxSymbolId); - dictionaryChunksAwaitCommit = true; - sizingBaseline = currentBatchMaxSymbolId; - } else if (!fits) { - throw new BatchTooLargeForCapException( - "single table block too large for server batch cap in mixed schema batch; " - + "the batch is retained for retry until a larger-cap node is reached"); - } - - int frameBaseline = sizingBaseline; - try { - for (int i = 0; i < blockCount; i++) { - boolean frameDefer = deferCommit || i + 1 < blockCount; - int messageSize = encodeSingleTableFrame(i, frameDefer, frameBaseline); - if (cap > 0 && messageSize > cap) { - throw new AssertionError("mixed frame exceeded preflight cap"); - } - persistNewSymbolsBeforePublish(); - QwpBufferWriter buffer = encoder.getBuffer(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); - activeBuffer.incrementRowCount(); - sealAndSwapBuffer(); - if (frameDefer) { - hasDeferredMessages = true; - } - advanceSentMaxSymbolId(); - if (deltaDictEnabled && currentBatchMaxSymbolId > frameBaseline) { - frameBaseline = currentBatchMaxSymbolId; - } - } - hasDeferredMessages = deferCommit; - if (!deferCommit) { - lastCommitBoundaryFsn = cursorEngine.publishedFsn(); - } - resetTableBuffersAfterFlush(); - } catch (Throwable t) { - if (dictionaryChunksAwaitCommit) { - commitOrphanedDictionaryChunks(t); - } - throw t; - } - } - - private boolean mixedFramesFit(int cap, int deltaBaseline) { - if (cap <= 0) { - return true; - } - int simBaseline = deltaBaseline; - for (int i = 0, n = flushTableBuffers.size(); i < n; i++) { - int messageSize = encodeSingleTableFrame(i, true, simBaseline); - if (messageSize > cap) { - return false; - } - if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { - simBaseline = currentBatchMaxSymbolId; - } - } - return true; - } - - private int encodeSingleTableFrame(int index, boolean deferCommit, int deltaBaseline) { - encoder.setDeferCommit(deferCommit); - QwpTableBuffer tableBuffer = flushTableBuffers.getQuick(index); - QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); - if (binding == null) { - encoder.beginMessage(1, globalSymbolDictionary, deltaBaseline, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - } else { - encoder.beginSchemaMessage(1, globalSymbolDictionary, deltaBaseline, currentBatchMaxSymbolId); - encoder.addSchemaTable(tableBuffer, binding.getTableId(), binding.getMetadataVersion()); - } - return encoder.finishMessage(); - } - - private boolean isMixedSchemaBatch(int tableCount) { - boolean schema = flushTableBuffers.getQuick(0).getSchemaBinding() != null; - for (int i = 1; i < tableCount; i++) { - if ((flushTableBuffers.getQuick(i).getSchemaBinding() != null) != schema) { - return true; - } - } - return false; - } - /** * Encodes the staged batch as one combined frame at {@code deltaBaseline}: * begins the message, appends every non-empty table, and records each table's @@ -5692,6 +5603,9 @@ private void sendRow() { if (pendingRowCount == 0) { firstPendingRowTimeNanos = System.nanoTime(); + // The batch's first row selects its wire contract; see + // bindingForEffectiveWrite. + isPendingBatchLegacy = currentTableBuffer.getSchemaBinding() == null; } pendingRowCount++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 339393e68..af5b9e5d2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -1854,7 +1854,7 @@ public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() } @Test - public void testPendingLegacyRowSurvivesUpgradeAndPublishesBeforeSchemaRow() throws Exception { + public void testPendingLegacyBatchStaysLegacyAcrossUpgradeAndNextBatchAdoptsSchema() throws Exception { assertMemoryLeak(() -> { int port = TestPorts.findUnusedPort(); SchemaHandler legacyHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); @@ -1887,17 +1887,22 @@ public void testPendingLegacyRowSurvivesUpgradeAndPublishesBeforeSchemaRow() thr Assert.assertTrue("schema-capable connection was not installed and servicing frames", schemaHandler.awaitPong()); sender.atNow(); + // A batch has one wire contract: the pending batch already holds + // legacy rows, so this row stays legacy although the supporting + // connection is up. The first row after the flush adopts schema mode. + sender.table("events").stringColumn("legacy_value", "C").atNow(); + sender.flush(); sender.table("events").uuidColumn("id", 3, 4).atNow(); sender.flush(); List frames = schemaHandler.awaitDataFrames(2); Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); - Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] - & QwpConstants.FLAG_DEFER_COMMIT) != 0); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); Assert.assertTrue((frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_DEFER_COMMIT); - new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B"); + new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B", "C"); FrameReader schema = new FrameReader(frames.get(1)); schema.schemaTable("events", 61, 62, 1, "id", QwpConstants.TYPE_UUID); Assert.assertEquals(0, schema.u8()); @@ -1939,12 +1944,16 @@ public void testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect() t Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + // The pending batch is legacy, so this row stays legacy too. The + // first row after the flush adopts schema mode. + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); sender.table("events").timestampColumn("ts", value).atNow(); sender.flush(); List frames = handler.awaitDataFrames(2); new FrameReader(frames.get(0)).legacyTimestampTable( - "events", "ts", 3_456_789L); + "events", "ts", 3_456_789L, 3_456_789L); new FrameReader(frames.get(1)).schemaTimestampTable( "events", 71, 72, "ts", 3_456_789_123L); Assert.assertEquals(1, handler.describeRequests.get()); @@ -2417,7 +2426,7 @@ private void legacyVarcharTable(String table, String column, String... values) { eof(); } - private void legacyTimestampTable(String table, String column, long expectedMicros) { + private void legacyTimestampTable(String table, String column, long... expectedMicros) { Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); Assert.assertEquals(QwpConstants.VERSION, u8()); Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); @@ -2426,13 +2435,15 @@ private void legacyTimestampTable(String table, String column, long expectedMicr Assert.assertEquals(0, varint()); Assert.assertEquals(0, varint()); Assert.assertEquals(table, string()); - Assert.assertEquals(1, varint()); + Assert.assertEquals(expectedMicros.length, varint()); Assert.assertEquals(1, varint()); Assert.assertEquals(column, string()); Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, u8()); Assert.assertEquals(0, u8()); Assert.assertEquals(0, u8()); - Assert.assertEquals(expectedMicros, i64()); + for (long micros : expectedMicros) { + Assert.assertEquals(micros, i64()); + } eof(); } diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 300a5e498..9e40ffe97 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -8019,3 +8019,27 @@ their fragmented-transport variant. Final server scratch is under uses `/mnt/pcie5/qwp-schema-aware-tests/client-final`, with no `/tmp` paths in its 295 current Surefire reports. No client or server production source changed in this decision. + +### Legacy-to-schema adoption moves to the batch boundary + +The 2026-09-17 complexity review ranked the duplicated mixed-family flush first. +Rather than fold it into the split path, this decision removes its cause. Row +boundary adoption was the only source of mixed legacy/schema batches. A batch +now has one wire contract, selected by its first committed row +(`isPendingBatchLegacy`); `bindingForEffectiveWrite()` keeps returning the +legacy contract while that batch is pending, and the first row after the flush +adopts schema mode. The upgrade stays one-way and is delayed by at most one +batch. `flushPendingRowsMixed()`, `mixedFramesFit()`, `encodeSingleTableFrame()` +and `isMixedSchemaBatch()` are deleted; the encoder is unchanged. This +supersedes the earlier "next effective row adopts schema mode" rule. The +trade-off: rows written after the handshake but inside the pending legacy batch +keep legacy conversion (for example microsecond `Instant` truncation). Which +rows preceded the handshake was already a timing race. + +The rewrite exposed an existing defect, reproduced on unmodified source: legacy +row, flush, supporting reconnect, then a row on the same table threw +`IllegalStateException: schema binding requires an empty, unbound table buffer` +from the column setter. `installSchemaBinding()` bound in place whenever the +buffer had no rows, but a flushed legacy buffer keeps its column layout. It now +binds in place only a buffer with no columns and replaces the buffer otherwise. +Both rewritten upgrade tests cover that path. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 5ad393c18..4f9ce8491 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -660,7 +660,7 @@ and report the capability mismatch under the existing reconnect policy. There is no send-time transformation, metadata stripping or downgrade encoder. A successful connection to an old server does not permanently prevent upgrade: a later connection that confirms schema support adopts schema mode at the next -row boundary. +batch boundary: the first row after the pending legacy batch flushes. | Client | Server | Behavior | |---|---|---| @@ -676,10 +676,13 @@ clients need no changes. Clients may be upgraded before servers. Once schema support has been confirmed, failover destinations must also support it; mixed old/new endpoints are not supported for that sender after the upgrade. -An existing legacy row finishes in legacy mode. The producer adopts schema -mode at the next actual row boundary, seals completed legacy rows and replaces -the inferred layout before binding new rows. Pending legacy blocks retain -their original types and flag-clear framing, even if encoded or sent later. A +A pending legacy batch finishes in legacy mode. A batch has one wire contract, +selected by its first committed row, because a QWP frame has one wire family: +while the batch holds legacy rows, later rows stay legacy. The producer adopts +schema mode on the first row after that batch flushes (explicitly or by +auto-flush) and replaces the inferred layout before binding new rows. Legacy +and schema blocks therefore never share a flush. Published legacy frames retain +their original types and flag-clear framing, even if sent later. A supporting server accepts those flag-clear blocks on a schema-negotiated connection; negotiation is required only for schema-flagged data. @@ -706,7 +709,8 @@ or option: retains the original values for later conversion. The mode is pinned by the first effective operation. If the handshake completes -mid-row, that row remains legacy and schema mode can start only on the next row. +mid-batch, that batch remains legacy and schema mode can start only on the first +row after it flushes. Consequently, handshake timing can change successful stored values as well as error timing: the legacy and schema contracts deliberately differ for cases such as sub-microsecond timestamps, overflow and source-null normalization. @@ -752,11 +756,11 @@ one-way upgrade and recovery. Do not also consult `hasEverConnected()` and do not add another negotiated-mode field. A supporting handshake may set the requirement immediately after the producer -observes false. That race is harmless and intentional: the current row has -already selected legacy behavior, while the next row observes the sticky true -value and enters schema mode. The existing partial-row check, retired table -buffers and mixed-frame flush preserve that boundary without a new lock or row -state. +observes false. That race is harmless and intentional: the current batch has +already selected legacy behavior, while the first row of the next batch observes +the sticky true value and enters schema mode. One producer-thread flag, set by +the batch's first committed row, preserves that boundary without a lock, and no +flush ever mixes wire families. No change belongs in `QwpSchemaCoordinator`, `CursorWebSocketSendLoop`, `CursorSendEngine`, the encoder, store-and-forward format or server. Update the @@ -2105,10 +2109,11 @@ without claiming general exactly-once delivery. extended SF replay on a negotiated connection, and reject flagged frames on unnegotiated connections before processing data. Restart with a mixed backlog and prove its original bytes, order and schema requirement are preserved. -- Upgrade from an old to a supporting server finishes any partial legacy row - unchanged and adopts schema mode at the next row boundary, including without - another `table()` call. Pending legacy blocks and SF remain unchanged and - ordered; new blocks use the extended layout. No send-time conversion occurs. +- Upgrade from an old to a supporting server finishes the pending legacy batch + unchanged and adopts schema mode on the first row after it flushes, including + on a table that already flushed legacy rows. Published legacy frames and SF + remain unchanged and ordered; new batches use the extended layout. No + send-time conversion occurs. - Reconnect and background replay re-negotiate support. After upgrade, an old endpoint receives no pending data and causes neither downgrade nor watermark advancement. The requirement survives reconnect even with a legacy-only From b94becd6356312a4b052858f6a51004efac83d5d Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 17 Sep 2026 16:33:40 +0200 Subject: [PATCH 23/51] Document batch-boundary schema adoption The ASYNC initial-connect javadoc in Sender still described the superseded rule: once schema support is confirmed, all future rows use strict schema lookup and only a row already in progress keeps its contract. The sender now selects one wire contract per batch, so rows added to a pending legacy batch stay legacy, and the first row after that batch flushes adopts schema mode. The javadoc states that rule. Two passages in the design document still named the row-boundary transition; they now name the batch boundary. Co-Authored-By: Claude Fable 5.1 --- core/src/main/java/io/questdb/client/Sender.java | 8 +++++--- design/schema-aware-sender.md | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index f49a3303c..a9ec63bbf 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -824,9 +824,11 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * therefore be reported later by the server, and values such as * nanosecond timestamps use their legacy representation. If recovered * data already requires schema support, writes instead wait for schema - * negotiation. Once support is confirmed, all future rows use strict - * schema lookup and validation, including across reconnects; a row - * already in progress keeps the contract under which it began. Transport + * negotiation. A batch keeps the contract under which its first row + * was written: rows added to a pending legacy batch stay legacy even + * after support is confirmed. The first row after that batch flushes + * (explicitly or by auto-flush) and all later rows use strict schema + * lookup and validation, including across reconnects. Transport * failures (unreachable or * dropped server) retry indefinitely and are never surfaced -- the * buffered rows are safe in SF and the server may still appear. A diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 4f9ce8491..ebea5e03d 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1379,7 +1379,8 @@ Do not extend the rule to other input families or targets by analogy. The table does not include missing-table/column inference or Sender lifecycle integration. Iteration 2.15 implements first-effective-write schema resolution, -designated timestamp row completion and row-boundary adoption. Iteration 2.18 +designated timestamp row completion and schema-mode adoption (since moved from +the row boundary to the batch boundary). Iteration 2.18 adds inference for the already-supported setter families, not the missing families in the conversion inventory. Preserve each public overload's null/no-op and duplicate rules: @@ -1872,8 +1873,8 @@ They do not change the compatibility contract. from the cursor's element type, so accepting a non-DOUBLE target would reproduce a server validation gap rather than a supported conversion. 28. **Reuse initial-connect policy only at the cold-start boundary.** Let - `ASYNC` create ordinary legacy rows until the first handshake, then use the - existing row-boundary transition to adopt schema mode. Keep `OFF` and + `ASYNC` create ordinary legacy rows until the first handshake, then adopt + schema mode at the next batch boundary. Keep `OFF` and `SYNC` strict, and let a recovered schema-required backlog override the startup exception. Do not add raw-value storage, a deferred converter, a second availability option or a downgrade path after schema confirmation. From 7f789df65899c26bbd6562545fb7f07f929a879a Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 17 Sep 2026 17:06:18 +0200 Subject: [PATCH 24/51] Let the schema snapshot own its column index QwpSchemaProtocol.decodePayload() built a case-insensitive column-name index to reject duplicate names and then discarded it. Every QwpSchemaBinding rebuilt the same index in its constructor, and hasSameRelevantTarget() resolved names in the refreshed schema with a third mechanism, a linear equalsIgnoreCase scan. The decoder now hands its index to QwpSchemaResponse, which exposes getColumnIndex() and owns hasSameRelevantTarget(). The binding drops its columns field and its constructor loop, and delegates the comparison to the snapshot. The index is never mutated after the decoder publishes it through a final field, so the I/O thread to user thread handoff stays safe. Each cached QwpSchemaResponse now retains its index for its lifetime. Before, only live bindings held one. QwpSchemaProtocolTest gains direct coverage for the case-insensitive lookup and for the target comparison, which only sender-level tests exercised before: absence on one or both sides, a retyped column, the extension-parameter flag, a moved or removed designated timestamp, and a schema-less RESULT_MISSING response. Co-Authored-By: Claude Fable 5.1 --- .../qwp/protocol/QwpSchemaBinding.java | 27 +------- .../qwp/protocol/QwpSchemaProtocol.java | 5 +- .../qwp/protocol/QwpSchemaResponse.java | 42 ++++++++++++- .../qwp/protocol/QwpSchemaProtocolTest.java | 62 +++++++++++++++++++ 4 files changed, 107 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 9f465b446..45047a52f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -35,7 +35,6 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Decimals; -import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; import io.questdb.client.std.NumericException; import io.questdb.client.std.Numbers; import io.questdb.client.std.bytes.DirectByteSlice; @@ -78,7 +77,6 @@ public final class QwpSchemaBinding { private static final int UUID_LENGTH = 36; private final QwpTableBuffer buffer; - private final LowerCaseCharSequenceIntHashMap columns = new LowerCaseCharSequenceIntHashMap(); private final QwpSchemaResponse schema; private final String tableName; private Decimal256 decimalTextScratch; @@ -108,9 +106,6 @@ public QwpSchemaBinding(QwpTableBuffer buffer, QwpSchemaResponse schema) { this.tableName = tableName; this.schema = schema; requireUsableSchema(); - for (int i = 0, n = schema.getColumnCount(); i < n; i++) { - columns.put(schema.getColumnName(i), i); - } this.buffer = buffer; buffer.attachSchemaBinding(this); } @@ -909,24 +904,7 @@ public long getMetadataVersion() { } public boolean hasSameRelevantTarget(QwpSchemaResponse other, CharSequence name) { - int current = name == null ? schema.getDesignatedIndex() : columns.get(name); - int next = -1; - if (name == null) { - next = other.getDesignatedIndex(); - } else { - for (int i = 0, n = other.getColumnCount(); i < n; i++) { - if (Chars.equalsIgnoreCase(other.getColumnName(i), name)) { - next = i; - break; - } - } - } - if (current < 0 || next < 0) { - return current < 0 && next < 0; - } - return (current == schema.getDesignatedIndex()) == (next == other.getDesignatedIndex()) - && schema.getColumnType(current) == other.getColumnType(next) - && schema.hasColumnExtensionParameters(current) == other.hasColumnExtensionParameters(next); + return schema.hasSameRelevantTarget(other, name); } private LineSenderSchemaException error( @@ -1867,8 +1845,7 @@ private int targetIndex(CharSequence name, String inputType) { if (name == null || !TableUtils.isValidColumnName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { throw error(INVALID_VALUE, name, inputType, null, "invalid column name"); } - int index = columns.get(name); - return index; + return schema.getColumnIndex(name); } private int targetType(int index, int inferredType) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java index 4b81efbcc..92c31129b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java @@ -130,7 +130,8 @@ private static QwpSchemaResponse decodePayload(long address, int length, boolean if (p != limit) { throw malformed("trailing bytes after schema result"); } - return new QwpSchemaResponse(requestId, result, -1, -1, -1, new QwpSchemaResponse.Column[0]); + return new QwpSchemaResponse(requestId, result, -1, -1, -1, new QwpSchemaResponse.Column[0], + new LowerCaseCharSequenceIntHashMap()); } if (length < KNOWN_FIXED_PAYLOAD_SIZE) { throw malformed("truncated known schema"); @@ -187,7 +188,7 @@ private static QwpSchemaResponse decodePayload(long address, int length, boolean if (p != limit) { throw malformed("trailing bytes after schema"); } - return new QwpSchemaResponse(requestId, result, tableId, metadataVersion, designatedIndex, columns); + return new QwpSchemaResponse(requestId, result, tableId, metadataVersion, designatedIndex, columns, columnNames); } public static byte[] encodeDescribe(long requestId, CharSequence tableName) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java index 9cbaf2efc..92285dc39 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java @@ -23,8 +23,15 @@ package io.questdb.client.cutlass.qwp.protocol; -/** Immutable response to a QWP schema DESCRIBE control request. */ +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; + +/** + * Immutable response to a QWP schema DESCRIBE control request. The snapshot + * owns the case-insensitive column-name index that the decoder builds while + * it validates name uniqueness, so consumers never rebuild it. + */ public final class QwpSchemaResponse { + private final LowerCaseCharSequenceIntHashMap columnIndexes; private final Column[] columns; private final int designatedIndex; private final int result; @@ -32,19 +39,33 @@ public final class QwpSchemaResponse { private final long metadataVersion; private final long requestId; - QwpSchemaResponse(long requestId, int result, int tableId, long metadataVersion, int designatedIndex, Column[] columns) { + QwpSchemaResponse( + long requestId, + int result, + int tableId, + long metadataVersion, + int designatedIndex, + Column[] columns, + LowerCaseCharSequenceIntHashMap columnIndexes + ) { this.requestId = requestId; this.result = result; this.tableId = tableId; this.metadataVersion = metadataVersion; this.designatedIndex = designatedIndex; this.columns = columns; + this.columnIndexes = columnIndexes; } public int getColumnCount() { return columns.length; } + /** Returns the index of the case-insensitively named column, or -1. */ + public int getColumnIndex(CharSequence name) { + return columnIndexes.get(name); + } + public String getColumnName(int index) { return columns[index].name; } @@ -83,6 +104,23 @@ public int getTableId() { return tableId; } + /** + * Returns whether {@code name} resolves to an equivalent conversion target + * in both snapshots: same presence, designated-timestamp role, column type, + * and extension-parameter flag. A null {@code name} selects the designated + * timestamp. Column position is irrelevant. + */ + public boolean hasSameRelevantTarget(QwpSchemaResponse other, CharSequence name) { + int current = name == null ? designatedIndex : getColumnIndex(name); + int next = name == null ? other.designatedIndex : other.getColumnIndex(name); + if (current < 0 || next < 0) { + return current < 0 && next < 0; + } + return (current == designatedIndex) == (next == other.designatedIndex) + && columns[current].type == other.columns[next].type + && columns[current].hasTypeParameters == other.columns[next].hasTypeParameters; + } + public boolean hasSchema() { return result == QwpSchemaProtocol.RESULT_KNOWN; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java index 1291802db..3c46655cb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java @@ -130,6 +130,68 @@ public void testEncodeDescribeUnicodeBoundariesAndRejectsInvalidNames() { assertEncodeRejected(1, "bad/name"); } + @Test + public void testGetColumnIndexIsCaseInsensitive() { + QwpSchemaResponse known = decode(knownResponse(1, 1, 1, 1, + column("Id", 5, new byte[0]), + column("métric🚀", 6, new byte[0]))); + Assert.assertEquals(0, known.getColumnIndex("id")); + Assert.assertEquals(0, known.getColumnIndex("ID")); + Assert.assertEquals(1, known.getColumnIndex("MÉTRIC🚀")); + Assert.assertEquals(-1, known.getColumnIndex("absent")); + + QwpSchemaResponse missing = decode(resultResponse(1, QwpSchemaProtocol.RESULT_MISSING)); + Assert.assertEquals(-1, missing.getColumnIndex("id")); + } + + @Test + public void testHasSameRelevantTarget() { + byte[] none = new byte[0]; + QwpSchemaResponse base = decode(knownResponse(1, 1, 1, 1, + column("v", 5, none), column("ts", 10, none))); + + // Position and unrelated columns do not matter; name case does not either. + QwpSchemaResponse moved = decode(knownResponse(2, 1, 2, 0, + column("TS", 10, none), column("extra", 7, none), column("V", 5, none))); + Assert.assertTrue(base.hasSameRelevantTarget(moved, "v")); + Assert.assertTrue(base.hasSameRelevantTarget(moved, "ts")); + Assert.assertTrue(base.hasSameRelevantTarget(moved, null)); + Assert.assertTrue(moved.hasSameRelevantTarget(base, "V")); + + // Absent on both sides is the same target; absent on one side is not. + Assert.assertTrue(base.hasSameRelevantTarget(moved, "absent")); + Assert.assertFalse(base.hasSameRelevantTarget(moved, "extra")); + Assert.assertFalse(moved.hasSameRelevantTarget(base, "extra")); + + QwpSchemaResponse retyped = decode(knownResponse(3, 1, 3, 1, + column("v", 6, none), column("ts", 10, none))); + Assert.assertFalse(base.hasSameRelevantTarget(retyped, "v")); + Assert.assertTrue(base.hasSameRelevantTarget(retyped, "ts")); + + QwpSchemaResponse parameterized = decode(knownResponse(4, 1, 4, 1, + column("v", 5, new byte[]{1}), column("ts", 10, none))); + Assert.assertFalse(base.hasSameRelevantTarget(parameterized, "v")); + + // The designated role moves from "ts" to "v": both named targets change, + // and so does the designated timestamp's type. + QwpSchemaResponse redesignated = decode(knownResponse(5, 1, 5, 0, + column("v", 5, none), column("ts", 10, none))); + Assert.assertFalse(base.hasSameRelevantTarget(redesignated, "v")); + Assert.assertFalse(base.hasSameRelevantTarget(redesignated, "ts")); + Assert.assertFalse(base.hasSameRelevantTarget(redesignated, null)); + + QwpSchemaResponse undesignated = decode(knownResponse(6, 1, 6, -1, + column("v", 5, none), column("ts", 10, none))); + Assert.assertFalse(base.hasSameRelevantTarget(undesignated, null)); + Assert.assertTrue(undesignated.hasSameRelevantTarget(undesignated, null)); + Assert.assertTrue(base.hasSameRelevantTarget(undesignated, "v")); + + QwpSchemaResponse missing = decode(resultResponse(7, QwpSchemaProtocol.RESULT_MISSING)); + Assert.assertFalse(base.hasSameRelevantTarget(missing, "v")); + Assert.assertTrue(missing.hasSameRelevantTarget(missing, "v")); + Assert.assertTrue(missing.hasSameRelevantTarget(missing, null)); + } + @Test public void testRejectsMalformedEnvelopesAndTrailingData() { byte[] valid = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); From 7a73e58e8d9354a045c9895c563e8009cd6ff8cf Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 17 Sep 2026 17:23:34 +0200 Subject: [PATCH 25/51] Share schema-binding test fixtures and reader The 21 QwpSchemaBinding*Test classes each carried a private wire Reader plus their own copies of the schema-response builders: 21 readers in 17 textual variants and two naming dialects (u8/i32/i64 vs byteValue/intValue/longValue), 8 variants of known(), 6 of column(), 4 of decode(). The copies were semantically identical, so a change to the schema envelope or the table header meant hand-editing up to 19 slightly different helpers. QwpSchemaTestFixtures now owns the schema-response builders (known, missing, schemaResult, column, parameterizedColumn, frame, decode), binding(), assertReason(), and the encoded-table header checks (tableHeader, skipTablePrefix, skipTableHeader). It writes the schema envelope by hand, independently of any production encoder. QwpTestWireReader is the single bounds-checked reader, with one naming dialect and a bounded varint; it never calls production decoders. The tests keep tableReader(), wireType(), targetType() and every conversion expectation, because those encode per-test intent. Two shared assertions are stricter than some of the copies they replace. assertReason() always asserts that the rejection is not retryable. tableHeader() always asserts the message magic, version, flags, and table count; six classes only skipped the header before. Four of the old readers read i32/i64 without a bounds check, and only one bounded its varint loop. The test count is unchanged at 139 across the binding classes. The change removes about 2,000 lines of test code. Co-Authored-By: Claude Fable 5.1 --- .../protocol/QwpSchemaBindingBinaryTest.java | 211 ++---- .../protocol/QwpSchemaBindingBooleanTest.java | 150 ++-- .../protocol/QwpSchemaBindingCharTest.java | 147 +--- .../QwpSchemaBindingDoubleArrayTest.java | 180 +---- .../QwpSchemaBindingFloatingDecimalTest.java | 123 +--- .../QwpSchemaBindingFloatingTest.java | 122 +--- .../QwpSchemaBindingFloatingTextTest.java | 171 +---- .../QwpSchemaBindingLongDecimalTest.java | 136 +--- .../QwpSchemaBindingLongTextTest.java | 201 ++---- .../QwpSchemaBindingLongTimestampTest.java | 142 +--- .../QwpSchemaBindingNativeDecimalTest.java | 186 +---- .../QwpSchemaBindingStringDateTest.java | 149 +--- .../QwpSchemaBindingStringDecimalTest.java | 136 +--- .../QwpSchemaBindingStringGeoHashTest.java | 129 +--- .../QwpSchemaBindingStringLong256Test.java | 118 +--- .../QwpSchemaBindingStringNumericTest.java | 198 ++---- .../QwpSchemaBindingStringTimestampTest.java | 152 +--- .../qwp/protocol/QwpSchemaBindingTest.java | 666 +++++++----------- .../protocol/QwpSchemaBindingTextTest.java | 181 +---- .../QwpSchemaBindingTimestampTextTest.java | 157 +---- .../QwpSchemaBindingUuidTextTest.java | 121 +--- .../qwp/protocol/QwpSchemaTestFixtures.java | 190 +++++ .../qwp/protocol/QwpTestWireReader.java | 140 ++++ 23 files changed, 1047 insertions(+), 3059 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java index 770188773..2e46eef5b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java @@ -10,8 +10,6 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; @@ -22,10 +20,11 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingBinaryTest { @@ -57,11 +56,11 @@ public void testConformanceCorpusUsesExactBinaryWire() throws Exception { throw new AssertionError(fields[1]); } buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); if ("".equals(fields[4])) { - Assert.assertEquals(fields[0], 1, reader.byteValue()); - Assert.assertEquals(fields[0], 1, reader.byteValue()); - Assert.assertEquals(fields[0], 0, reader.intValue()); + Assert.assertEquals(fields[0], 1, reader.u8()); + Assert.assertEquals(fields[0], 1, reader.u8()); + Assert.assertEquals(fields[0], 0, reader.i32()); } else { byte[] expected = bytes(fields[4]); if ("binary_all_256".equals(fields[0])) { @@ -69,9 +68,9 @@ public void testConformanceCorpusUsesExactBinaryWire() throws Exception { assertAllByteValues(expected); assertAllByteValues(bytes(fields[5])); } - Assert.assertEquals(fields[0], 0, reader.byteValue()); - Assert.assertEquals(fields[0], 0, reader.intValue()); - Assert.assertEquals(fields[0], expected.length, reader.intValue()); + Assert.assertEquals(fields[0], 0, reader.u8()); + Assert.assertEquals(fields[0], 0, reader.i32()); + Assert.assertEquals(fields[0], expected.length, reader.i32()); Assert.assertArrayEquals(fields[0], expected, reader.bytes(expected.length)); } Assert.assertEquals(fields[0], encoder.getBuffer().getPosition(), reader.position()); @@ -110,10 +109,10 @@ public void testAllBinaryOverloadsCopyBeforeReturning() throws Exception { binding.binaryColumn("value", 0, 0); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 4, 1); - Assert.assertEquals(0, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 4, 1); + Assert.assertEquals(0, reader.u8()); for (int offset : new int[]{0, 2, 4, 6, 6}) { - Assert.assertEquals(offset, reader.intValue()); + Assert.assertEquals(offset, reader.i32()); } Assert.assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6}, reader.bytes(6)); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); @@ -137,22 +136,22 @@ public void testDuplicatePrecedesBinaryArgumentValidation() throws Exception { .binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1) .stringColumn("value", new ThrowingCharSequence(new TestCharSequenceException())); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(42, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(42, reader.u8()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); binding.stringColumn("value", "x").binaryColumn("value", (byte[]) null); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals('x', reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals('x', reader.u8()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -189,17 +188,17 @@ public void testBitmapCrossesSixtyFourRowsWithPresentAndOmittedValues() throws E } buffer.nextRow(); } - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 130, 1); - Assert.assertEquals(1, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 130, 1); + Assert.assertEquals(1, reader.u8()); for (int i = 0; i < 16; i++) { - Assert.assertEquals(0xaa, reader.byteValue()); + Assert.assertEquals(0xaa, reader.u8()); } - Assert.assertEquals(0x02, reader.byteValue()); + Assert.assertEquals(0x02, reader.u8()); for (int i = 0; i <= 65; i++) { - Assert.assertEquals(i, reader.intValue()); + Assert.assertEquals(i, reader.i32()); } for (int row = 0; row < 130; row += 2) { - Assert.assertEquals(row & 0xff, reader.byteValue()); + Assert.assertEquals(row & 0xff, reader.u8()); } Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } @@ -228,15 +227,15 @@ public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, 2, 3); Assert.assertEquals("a", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); Assert.assertEquals("existing_b", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); Assert.assertEquals("c", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); assertSparseBinary(reader, 2, 1); assertSparseBinary(reader, 2, 2); assertSparseBinary(reader, 1, 3); @@ -262,10 +261,10 @@ public void testStringAppendIsExceptionAtomicForVarcharRuntimeAndError() throws rollback(buffer); binding.stringColumn("value", "c"); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, 1, QwpConstants.TYPE_VARCHAR); - Assert.assertEquals(0, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); for (int offset : new int[]{0, 1, 2}) { - Assert.assertEquals(offset, reader.intValue()); + Assert.assertEquals(offset, reader.i32()); } Assert.assertArrayEquals(new byte[]{'a', 'c'}, reader.bytes(2)); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); @@ -284,11 +283,11 @@ public void testResetClearAndSchemaGuards() throws Exception { buffer.reset(); binding.binaryColumn("value", new byte[]{8}); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(8, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(8, reader.u8()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); buffer.clear(); Assert.assertThrows(IllegalStateException.class, () -> binding.binaryColumn("value", new byte[]{1})); @@ -308,12 +307,12 @@ public void testResetClearAndSchemaGuards() throws Exception { }); } - private static void assertSparseBinary(Reader reader, int bitmap, int value) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(bitmap, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(value, reader.byteValue()); + private static void assertSparseBinary(QwpTestWireReader reader, int bitmap, int value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(value, reader.u8()); } private static void assertAllByteValues(byte[] bytes) { @@ -369,127 +368,19 @@ private static String utf16(String hex) { return new String(chars); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { return tableReader(encoder, size, rows, columns, QwpConstants.TYPE_BINARY); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns, int wireType) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns, int wireType) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, rows, columns); Assert.assertEquals("value", reader.string()); - Assert.assertEquals(wireType, reader.byteValue()); + Assert.assertEquals(wireType, reader.u8()); return reader; } - private static void skipTablePrefix(Reader reader, int rows, int columns) { - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - } - - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1) - .put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN).putInt(QwpConstants.MAGIC_MESSAGE) - .put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(length).put(payload.array()); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - put(address, frame.array()); - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int byteValue() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 255; - } - - private byte[] bytes(int count) { - Assert.assertTrue(position + count <= limit); - byte[] bytes = new byte[count]; - for (int i = 0; i < count; i++) { - bytes[i] = (byte) byteValue(); - } - return bytes; - } - - private int intValue() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long longValue() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int count) { - Assert.assertTrue(position + count <= limit); - position += count; - } - - private String string() { - return new String(bytes(varint()), StandardCharsets.UTF_8); - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = byteValue(); - result |= (value & 127) << shift; - shift += 7; - } while ((value & 128) != 0); - return result; - } - } - private static final class ThrowingCharSequence implements CharSequence { private final Throwable failure; private final int failureIndex; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java index 1bf7185ed..1e0136c89 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java @@ -10,21 +10,19 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingBooleanTest { @@ -50,8 +48,8 @@ public void testBooleanCorpusUsesExactTargetWire() throws Exception { Assert.assertTrue(fields[1], "TRUE".equals(fields[1]) || "FALSE".equals(fields[1])); binding.boolColumn("value", "TRUE".equals(fields[1])); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(fields[3]), 1); - Assert.assertEquals(0, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(fields[3]), 1); + Assert.assertEquals(0, reader.u8()); assertValue(reader, fields[3], fields[4]); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } catch (AssertionError e) { @@ -86,13 +84,13 @@ public void testStringCorpusParsesStrictAsciiAndUsesBooleanWire() throws Excepti } else { binding.stringColumn("value", value); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 1); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 1); if (value == null) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); } else { - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(Integer.parseInt(fields[2]), reader.byteValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(fields[2]), reader.u8()); } Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } @@ -117,8 +115,8 @@ public void testBooleanPackingPastEightAndSixtyFourWithOmissions() throws Except } buffer.nextRow(); } - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 137); - Assert.assertEquals(1, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 137); + Assert.assertEquals(1, reader.u8()); byte[] nulls = new byte[18]; for (int i = 0; i < 17; i++) nulls[i] = (byte) 0xaa; Assert.assertArrayEquals(nulls, reader.bytes(nulls.length)); @@ -130,8 +128,8 @@ public void testBooleanPackingPastEightAndSixtyFourWithOmissions() throws Except buffer.reset(); for (int row = 0; row < 70; row++) buffer.nextRow(); - Reader allNull = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 70); - Assert.assertEquals(1, allNull.byteValue()); + QwpTestWireReader allNull = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 70); + Assert.assertEquals(1, allNull.u8()); byte[] allNulls = new byte[9]; for (int i = 0; i < 8; i++) allNulls[i] = (byte) 0xff; allNulls[8] = 0x3f; @@ -156,19 +154,19 @@ public void testDuplicateRollbackFailedRowAndLifecycle() throws Exception { binding.boolColumn("c", true); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, 2, 2); Assert.assertEquals("a", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reader.u8()); Assert.assertEquals("c", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_INT, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); Assert.assertEquals(size, reader.position()); buffer.reset(); @@ -177,17 +175,17 @@ public void testDuplicateRollbackFailedRowAndLifecycle() throws Exception { mutable.replace(0, mutable.length(), "false"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); reset.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reset, 1, 2); Assert.assertEquals("a", reset.string()); - Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reset.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reset.u8()); Assert.assertEquals("c", reset.string()); - Assert.assertEquals(QwpConstants.TYPE_INT, reset.byteValue()); - Assert.assertEquals(0, reset.byteValue()); - Assert.assertEquals(1, reset.byteValue()); - Assert.assertEquals(1, reset.byteValue()); - Assert.assertEquals(1, reset.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_INT, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); Assert.assertEquals(resetSize, reset.position()); buffer.clear(); assertIllegalState(() -> binding.boolColumn("a", true)); @@ -237,74 +235,34 @@ public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { } } - private static void assertValue(Reader reader, String type, String expected) { + private static void assertValue(QwpTestWireReader reader, String type, String expected) { switch (type) { - case "BOOLEAN": reader.assertByte(Integer.parseInt(expected)); break; - case "BYTE": reader.assertByte(Integer.parseInt(expected)); break; - case "SHORT": Assert.assertEquals(Integer.parseInt(expected), reader.shortValue()); break; - case "INT": Assert.assertEquals(Integer.parseInt(expected), reader.intValue()); break; - case "LONG": Assert.assertEquals(Long.parseLong(expected), reader.longValue()); break; - case "FLOAT": Assert.assertEquals(Float.floatToRawIntBits(Float.parseFloat(expected)), Float.floatToRawIntBits(reader.floatValue())); break; - case "DOUBLE": Assert.assertEquals(Double.doubleToRawLongBits(Double.parseDouble(expected)), Double.doubleToRawLongBits(reader.doubleValue())); break; + case "BOOLEAN": Assert.assertEquals(Integer.parseInt(expected), reader.u8()); break; + case "BYTE": Assert.assertEquals(Integer.parseInt(expected), reader.u8()); break; + case "SHORT": Assert.assertEquals(Integer.parseInt(expected), reader.i16()); break; + case "INT": Assert.assertEquals(Integer.parseInt(expected), reader.i32()); break; + case "LONG": Assert.assertEquals(Long.parseLong(expected), reader.i64()); break; + case "FLOAT": Assert.assertEquals(Float.floatToRawIntBits(Float.parseFloat(expected)), Float.floatToRawIntBits(reader.f32())); break; + case "DOUBLE": Assert.assertEquals(Double.doubleToRawLongBits(Double.parseDouble(expected)), Double.doubleToRawLongBits(reader.f64())); break; case "VARCHAR": byte[] bytes = hex(expected); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(bytes.length, reader.intValue()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(bytes.length, reader.i32()); Assert.assertArrayEquals(bytes, reader.bytes(bytes.length)); break; default: throw new AssertionError("unknown wire type: " + type); } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { return column(name, type, new byte[0]); } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) length += column.length; - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) payload.put(column); - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(length).put(payload.array()); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, rows, 1); Assert.assertEquals("value", reader.string()); - Assert.assertEquals(type, reader.byteValue()); + Assert.assertEquals(type, reader.u8()); return reader; } - private static void skipTablePrefix(Reader reader, int rows, int columns) { - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - } - private static int targetType(String value) { switch (value) { case "BOOLEAN": return ColumnType.BOOLEAN; @@ -349,24 +307,4 @@ private static String utf16(String value) { private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; private final int limit; private int position; - private Reader(long address, int limit) { this.address = address; this.limit = limit; } - private void assertByte(int value) { Assert.assertEquals(value, byteValue()); } - private int byteValue() { Assert.assertTrue(position < limit); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } - private byte[] bytes(int length) { Assert.assertTrue(position + length <= limit); byte[] value = new byte[length]; for (int i = 0; i < length; i++) value[i] = (byte) byteValue(); return value; } - private short shortValue() { Assert.assertTrue(position + 2 <= limit); short v = Unsafe.getUnsafe().getShort(address + position); position += 2; return v; } - private int intValue() { Assert.assertTrue(position + 4 <= limit); int v = Unsafe.getUnsafe().getInt(address + position); position += 4; return v; } - private long longValue() { Assert.assertTrue(position + 8 <= limit); long v = Unsafe.getUnsafe().getLong(address + position); position += 8; return v; } - private float floatValue() { return Float.intBitsToFloat(intValue()); } - private double doubleValue() { return Double.longBitsToDouble(longValue()); } - private int position() { return position; } - private void skip(int length) { Assert.assertTrue(position + length <= limit); position += length; } - private String string() { return new String(bytes(varint()), StandardCharsets.UTF_8); } - private int varint() { int r=0,s=0,v; do { v=byteValue(); r|=(v&0x7f)<".equals(fields[2])) { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); @@ -101,7 +101,7 @@ public void testCorpusUsesExactNativeCharWire() throws Exception { Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Integer.parseInt(fields[3]), reader.u16()); } - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); } catch (AssertionError e) { throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); } @@ -129,7 +129,7 @@ public void testDuplicateOmissionRollbackResetAndInference() { binding.stringColumn("c", "Z"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 3, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 3, 2); Assert.assertEquals("a", reader.string()); Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); Assert.assertEquals("c", reader.string()); @@ -140,13 +140,13 @@ public void testDuplicateOmissionRollbackResetAndInference() { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(3, reader.u8()); Assert.assertEquals('Z', reader.u16()); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); buffer.reset(); binding.stringColumn("a", "Q"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableHeader(encoder, resetSize, 1, 2); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); Assert.assertEquals("a", reset.string()); Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); Assert.assertEquals("c", reset.string()); @@ -155,7 +155,7 @@ public void testDuplicateOmissionRollbackResetAndInference() { Assert.assertEquals('Q', reset.u16()); Assert.assertEquals(1, reset.u8()); Assert.assertEquals(1, reset.u8()); - Assert.assertEquals(resetSize, reset.position); + Assert.assertEquals(resetSize, reset.position()); } try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { @@ -194,46 +194,6 @@ public void testUnsupportedAndDesignatedTargetsRemainRejected() { } } - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) - .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - private static String fromUtf16Hex(String hex) { StringBuilder value = new StringBuilder(hex.length() / 4); for (int i = 0; i < hex.length(); i += 4) { @@ -242,85 +202,10 @@ private static String fromUtf16Hex(String hex) { return value.toString(); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { - Reader reader = tableHeader(encoder, size, 1, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); Assert.assertEquals("value", reader.string()); Assert.assertEquals(type, reader.u8()); return reader; } - - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); - Assert.assertEquals(QwpConstants.VERSION, reader.u8()); - Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); - Assert.assertEquals(1, reader.u16()); - Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long i64() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int u16() { - return u8() | (u8() << 8); - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java index a889eb4fc..50278f69f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java @@ -12,18 +12,15 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; - +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingDoubleArrayTest { @@ -50,7 +47,7 @@ public void testAllPublicRepresentationsUseExactTargetWire() throws Exception { buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 1, 4); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 4); columnDef(reader, "a1"); columnDef(reader, "a2"); columnDef(reader, "a3"); @@ -85,7 +82,7 @@ public void testNullRepresentationsAreNoopsAndWrapperRankTracksReshape() throws buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 1, 1, -1, -1); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1, -1, -1); columnDef(reader, "value"); Assert.assertEquals(0, reader.u8()); rawArray(reader, 1, new int[]{1}, new double[]{42.0}); @@ -128,7 +125,7 @@ public void testKnownRankDuplicateRollbackAndShapeValidation() throws Exception binding.doubleArray("v", new double[0]); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 2, 1); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1); columnDef(reader, "v"); Assert.assertEquals(0, reader.u8()); rawArray(reader, 1, new int[]{1}, new double[]{1.0}); @@ -156,7 +153,7 @@ public void testMissingSchemaPinsFirstEffectiveRank() throws Exception { binding.doubleArray("v", new double[][]{{4.0}, {5.0}}); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 2, 1, -1, -1); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1, -1, -1); columnDef(reader, "v"); Assert.assertEquals(0, reader.u8()); rawArray(reader, 2, new int[]{1, 2}, new double[]{1.0, 2.0}); @@ -196,7 +193,7 @@ public void testRejectsWrongElementTypeWeakRankParametersAndDesignatedColumn() { } } - private static void array(Reader reader, int[] shape, double... values) { + private static void array(QwpTestWireReader reader, int[] shape, double... values) { Assert.assertEquals(0, reader.u8()); rawArray(reader, shape.length, shape, values); } @@ -205,65 +202,12 @@ private static int arrayType(int dimensions) { return ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions); } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return binding(buffer, -1, columns); - } - - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type) - .putShort((short) params.length).put(params).array(); - } - - private static void columnDef(Reader reader, String name) { + private static void columnDef(QwpTestWireReader reader, String name) { Assert.assertEquals(name, reader.string()); Assert.assertEquals(QwpConstants.TYPE_DOUBLE_ARRAY, reader.u8()); } - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static void rawArray(Reader reader, int expectedDims, int[] shape, double[] values) { + private static void rawArray(QwpTestWireReader reader, int expectedDims, int[] shape, double[] values) { Assert.assertEquals(expectedDims, reader.u8()); for (int dimension : shape) { Assert.assertEquals(dimension, reader.i32()); @@ -272,104 +216,4 @@ private static void rawArray(Reader reader, int expectedDims, int[] shape, doubl Assert.assertEquals(Double.doubleToRawLongBits(value), reader.i64()); } } - - private static void assertReason( - LineSenderSchemaException.Reason reason, - Runnable action, - String... messageParts - ) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(reason, error.getReason()); - for (String part : messageParts) { - Assert.assertTrue(error.getMessage(), error.getMessage().contains(part)); - } - } - - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - return tableHeader(encoder, size, rows, columns, 1, 1); - } - - private static Reader tableHeader( - QwpWebSocketEncoder encoder, - int size, - int rows, - int columns, - int tableId, - long version - ) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - reader.skip(QwpConstants.HEADER_SIZE); - Assert.assertEquals("t", reader.string()); - if (tableId < 0) { - Assert.assertEquals(0, reader.u8()); - } else { - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(tableId, reader.i32()); - Assert.assertEquals(version, reader.i64()); - } - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position <= limit - Integer.BYTES); - int value = Unsafe.getUnsafe().getInt(address + position); - position += Integer.BYTES; - return value; - } - - private long i64() { - Assert.assertTrue(position <= limit - Long.BYTES); - long value = Unsafe.getUnsafe().getLong(address + position); - position += Long.BYTES; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - for (; shift < 32; shift += 7) { - int b = u8(); - result |= (b & 0x7f) << shift; - if ((b & 0x80) == 0) { - return result; - } - } - throw new AssertionError("unterminated varint"); - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java index 910129744..0ba41eafd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java @@ -10,10 +10,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; @@ -21,10 +18,11 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingFloatingDecimalTest { @@ -94,7 +92,7 @@ public void testDuplicateNullRollbackAndOmissionLifecycle() throws Exception { buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, QwpConstants.TYPE_DECIMAL64, 4); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_DECIMAL64, 4); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(0x0a, reader.u8()); Assert.assertEquals(1, reader.u8()); @@ -105,7 +103,7 @@ public void testDuplicateNullRollbackAndOmissionLifecycle() throws Exception { buffer.reset(); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL64, 1); + QwpTestWireReader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL64, 1); Assert.assertEquals(1, reset.u8()); Assert.assertEquals(1, reset.u8()); Assert.assertEquals(1, reset.u8()); @@ -131,12 +129,6 @@ private static void assertInvalid(Vector vector) throws Exception { } } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(error.getMessage(), reason, error.getReason()); - Assert.assertFalse(error.getMessage(), error.isRetryable()); - } - private static void assertVector(Vector vector) throws Exception { try { if (vector.invalid()) { @@ -153,7 +145,7 @@ private static void assertVector(Vector vector) throws Exception { Assert.assertEquals(QwpConstants.FLAG_SCHEMA, Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); - Reader reader = tableReader(encoder, size, wireType(vector.targetType), 1); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType), 1); if (vector.isNull()) { Assert.assertEquals(vector.caseId, 1, reader.u8()); Assert.assertEquals(vector.caseId, 1, reader.u8()); @@ -172,52 +164,13 @@ private static void assertVector(Vector vector) throws Exception { } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static QwpSchemaResponse known(byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) -1).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - private static long parseHex(String value) { Assert.assertNotEquals("-", value); return Long.parseUnsignedLong(value, 16); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.u8()); @@ -267,66 +220,6 @@ private static int wireLongCount(String target) { return 1; } - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long i64() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } - private static final class Vector { private final String caseId; private final long[] expectedLimbs = new long[4]; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java index aa47bad4b..d2f1e7801 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java @@ -10,21 +10,19 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingFloatingTest { @@ -51,14 +49,14 @@ public void testConformanceCorpusUsesExactTargetWire() throws Exception { } else { append.run(); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[4]), 1, 1); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[4]), 1, 1); if ("".equals(f[5])) { Assert.assertEquals(f[0], "BYTE".equals(f[3]) || "SHORT".equals(f[3]) ? "0" : "NULL", f[6]); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); } else { - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); Assert.assertArrayEquals(f[0], hex(f[5]), reader.bytes(hex(f[5]).length)); } Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); @@ -83,10 +81,10 @@ public void testNaNIsMissingWithOtherNullAndPresentValues() throws Exception { buffer.nextRow(); binding.doubleColumn("value", -0.0d); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_DOUBLE, 3, 1); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(3, reader.byteValue()); - Assert.assertEquals(0x8000000000000000L, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_DOUBLE, 3, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals(0x8000000000000000L, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -102,18 +100,18 @@ public void testDuplicatePrecedesInvalidAndUnsupportedConversion() throws Except binding.uuidColumn("uuid", 1, 2).floatColumn("uuid", Float.NaN); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, 1, 2); Assert.assertEquals("n", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_LONG, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); Assert.assertEquals("uuid", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_UUID, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -136,19 +134,19 @@ public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws binding.floatColumn("c", 3); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, 2, 2); Assert.assertEquals("a", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_FLOAT, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_FLOAT, reader.u8()); Assert.assertEquals("c", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_LONG, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(Float.floatToRawIntBits(1.5f), reader.intValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(3, reader.longValue()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(Float.floatToRawIntBits(1.5f), reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -164,9 +162,9 @@ public void testResetClearAndStaleBinding() throws Exception { buffer.reset(); binding.floatColumn("value", -0.0f); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_FLOAT, 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0x80000000, reader.intValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_FLOAT, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x80000000, reader.i32()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); buffer.clear(); assertIllegalState(() -> binding.floatColumn("value", 1)); @@ -219,49 +217,15 @@ private static void append(QwpSchemaBinding binding, String inputType, String bi } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { return column(name, type, new byte[0]); } - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) length += column.length; - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) payload.put(column); - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(length).put(payload.array()); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, rows, columns); Assert.assertEquals("value", reader.string()); - Assert.assertEquals(type, reader.byteValue()); + Assert.assertEquals(type, reader.u8()); return reader; } - private static void skipTablePrefix(Reader reader, int rows, int columns) { - Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(rows, reader.varint()); Assert.assertEquals(columns, reader.varint()); - } - private static int targetType(String value) { switch (value) { case "BYTE": return ColumnType.BYTE; case "SHORT": return ColumnType.SHORT; @@ -298,20 +262,4 @@ private static void assertInvalidContext(QwpSchemaBinding binding, QwpTableBuffe rollback(buffer); } private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; private final int limit; private int position; - private Reader(long address, int limit) { this.address=address; this.limit=limit; } - private int byteValue() { Assert.assertTrue(position".equals(fields[4])) { Assert.assertEquals("", fields[5]); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - if (wireType == QwpConstants.TYPE_VARCHAR) Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) Assert.assertEquals(0, reader.i32()); else Assert.assertEquals(0, reader.varint()); } else { byte[] expected = hex(fields[4]); Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); if (wireType == QwpConstants.TYPE_VARCHAR) { - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); Assert.assertArrayEquals(expected, reader.bytes(expected.length)); } else { Assert.assertEquals(1, reader.varint()); @@ -102,12 +101,12 @@ public void testDuplicateOmissionRollbackAndReset() throws Exception { buffer.nextRow(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, wireType, 3); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(5, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); if (wireType == QwpConstants.TYPE_VARCHAR) { - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(2, reader.intValue()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.bytes(2)); } else { Assert.assertEquals(1, reader.varint()); @@ -133,20 +132,20 @@ public void testDuplicateOmissionRollbackAndReset() throws Exception { buffer.nextRow(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, QwpConstants.TYPE_VARCHAR, 3, 2); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_VARCHAR, 3, 2); Assert.assertEquals("a", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); Assert.assertEquals("c", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(6, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(2, reader.intValue()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(6, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); Assert.assertArrayEquals(new byte[]{'1', '0'}, reader.bytes(2)); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(5, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(2, reader.intValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); Assert.assertArrayEquals(new byte[]{'3', '0'}, reader.bytes(2)); Assert.assertEquals(size, reader.position()); } @@ -163,8 +162,8 @@ public void testDuplicateOmissionRollbackAndReset() throws Exception { binding.longColumn("value", 42); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_SYMBOL, 3); - Assert.assertEquals(0, reset.byteValue()); + QwpTestWireReader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reset.u8()); Assert.assertEquals(2, reset.varint()); Assert.assertArrayEquals(new byte[]{'4', '2'}, reset.stringBytes()); Assert.assertArrayEquals(new byte[]{'4', '3'}, reset.stringBytes()); @@ -186,16 +185,16 @@ public void testMissingSchemaInfersNativeLong() { Assert.assertEquals(1, buffer.getColumnCount()); Assert.assertEquals(QwpConstants.TYPE_LONG, buffer.getColumnDefs()[0].getTypeCode()); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(1, reader.varint()); Assert.assertEquals(1, reader.varint()); Assert.assertEquals("value", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_LONG, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(42, reader.longValue()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(42, reader.i64()); Assert.assertEquals(size, reader.position()); } } @@ -230,7 +229,7 @@ public void testOwnerBackedSymbolUsesGlobalDictionary() throws Exception { encoder.beginSchemaMessage(1, mirror, -1, 2); encoder.addSchemaTable(buffer, binding.getTableId(), binding.getMetadataVersion()); int size = encoder.finishMessage(); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals(0, reader.varint()); Assert.assertEquals(3, reader.varint()); @@ -238,7 +237,7 @@ public void testOwnerBackedSymbolUsesGlobalDictionary() throws Exception { Assert.assertArrayEquals("42".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); Assert.assertArrayEquals("43".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 3); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(1, reader.varint()); Assert.assertEquals(2, reader.varint()); Assert.assertEquals(1, reader.varint()); @@ -249,78 +248,22 @@ public void testOwnerBackedSymbolUsesGlobalDictionary() throws Exception { }); } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return new QwpSchemaBinding(buffer, response((byte) QwpSchemaProtocol.RESULT_KNOWN, columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static byte[] frame(byte[] payload) { - return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload).array(); - } - - private static QwpSchemaResponse missing() { - ByteBuffer payload = ByteBuffer.allocate(1 + 8 + 1).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING); - return decodeResponse(payload.array()); - } - - private static QwpSchemaResponse response(byte result, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) length += column.length; - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put(result) - .putInt(result == QwpSchemaProtocol.RESULT_KNOWN ? 1 : -1) - .putLong(result == QwpSchemaProtocol.RESULT_KNOWN ? 1 : -1) - .putShort((short) -1).putShort((short) columns.length); - for (byte[] column : columns) payload.put(column); - return decodeResponse(payload.array()); - } - - private static QwpSchemaResponse decodeResponse(byte[] payload) { - byte[] frame = frame(payload); - long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.length; i++) Unsafe.getUnsafe().putByte(address + i, frame[i]); - return QwpSchemaProtocol.decodeResponse(address, frame.length); - } finally { - Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { return tableReader(encoder, size, wireType, rows, 1); } - private static void skipTableHeader(Reader reader, byte wireType, int rows) { - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(1, reader.varint()); - Assert.assertEquals("value", reader.string()); - Assert.assertEquals(wireType, reader.byteValue()); - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.longValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); Assert.assertEquals(rows, reader.varint()); Assert.assertEquals(columns, reader.varint()); if (columns == 1) { Assert.assertEquals("value", reader.string()); - Assert.assertEquals(wireType, reader.byteValue()); + Assert.assertEquals(wireType, reader.u8()); } return reader; } @@ -341,60 +284,4 @@ private static byte[] hex(String value) { for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); return bytes; } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(reason, error.getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int byteValue() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private byte[] bytes(int length) { - byte[] value = new byte[length]; - for (int i = 0; i < length; i++) value[i] = (byte) byteValue(); - return value; - } - - private int intValue() { - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long longValue() { - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - private int position() { return position; } - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - private String string() { return new String(stringBytes(), StandardCharsets.UTF_8); } - private byte[] stringBytes() { return bytes(varint()); } - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = byteValue(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java index 6ae74134a..5cfa9e527 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java @@ -10,21 +10,20 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingLongTimestampTest { @@ -53,7 +52,7 @@ public void testCorpusUsesExactTargetUnitWire() throws Exception { binding.longColumn("value", value); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, "value", wireType); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", wireType); if ("".equals(fields[4])) { Assert.assertEquals("true", fields[5]); Assert.assertEquals(1, reader.u8()); @@ -92,7 +91,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { buffer.nextRow(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 3, "value", wireType); + QwpTestWireReader reader = tableReader(encoder, size, 3, "value", wireType); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(5, reader.u8()); Assert.assertEquals(0, reader.u8()); @@ -103,7 +102,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { binding.longColumn("value", Long.MAX_VALUE); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableReader(encoder, resetSize, 1, "value", wireType); + QwpTestWireReader reset = tableReader(encoder, resetSize, 1, "value", wireType); Assert.assertEquals(0, reset.u8()); Assert.assertEquals(0, reset.u8()); Assert.assertEquals(Long.MAX_VALUE, reset.i64()); @@ -127,7 +126,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { binding.longColumn("c", 4); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 2, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); Assert.assertEquals("a", reader.string()); Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); Assert.assertEquals("c", reader.string()); @@ -152,7 +151,7 @@ public void testNativeInferenceAndNamedDesignatedRejection() { binding.longColumn("value", Long.MIN_VALUE); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(0, reader.u8()); @@ -170,69 +169,13 @@ public void testNativeInferenceAndNamedDesignatedRejection() { } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { - Reader reader = tableHeader(encoder, size, rows, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); Assert.assertEquals(name, reader.string()); Assert.assertEquals(type, reader.u8()); return reader; } - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - reader.skip(QwpConstants.HEADER_SIZE); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - private static int targetType(String value) { return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; } @@ -240,65 +183,4 @@ private static int targetType(String value) { private static byte wireType(String value) { return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long i64() { - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java index 4e9dbc912..b6bb8813f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java @@ -10,24 +10,24 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingNativeDecimalTest { @@ -66,7 +66,7 @@ public void testCorpusUsesExactTargetDecimalWireAndPreservesCaller() throws Exce append(binding, value); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, wireType(f[7])); + QwpTestWireReader reader = tableReader(encoder, size, wireType(f[7])); Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Integer.parseInt(f[9]), reader.u8()); assertExpectedLimbs(reader, f); @@ -97,7 +97,7 @@ public void testDecimalTextOverloadUsesEveryExactTargetWidth() throws Exception buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, wireTypeForPrecision(precision)); + QwpTestWireReader reader = tableReader(encoder, size, wireTypeForPrecision(precision)); Assert.assertEquals(0, reader.u8()); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(12, reader.i64()); @@ -139,7 +139,7 @@ public void testDecimalTextOverloadSuppressesDuplicatesAndRollsBackInvalidRows() buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 3, 1); + QwpTestWireReader reader = tableHeader(encoder, size, 3, 1); decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(0x06, reader.u8()); @@ -168,7 +168,7 @@ public void testDecimalTextOverloadValidatesTargetAfterDuplicateBeforeParsing() buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 1, 1); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); Assert.assertEquals("value", reader.string()); Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); Assert.assertEquals(0, reader.u8()); @@ -192,7 +192,7 @@ public void testDecimalTextOverloadMissingInfersDecimal256AtNaturalScale() throw buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256, 2); + QwpTestWireReader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256, 2); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(2, reader.u8()); @@ -215,7 +215,7 @@ public void testDecimalTextOverloadMissingSpecialInfersDecimal256ScaleZeroNull() buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256); + QwpTestWireReader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(0, reader.u8()); @@ -225,7 +225,7 @@ public void testDecimalTextOverloadMissingSpecialInfersDecimal256ScaleZeroNull() binding.decimalColumn("value", "5.678", new Decimal256()); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = unknownTableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL256); + QwpTestWireReader reset = unknownTableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL256); Assert.assertEquals(0, reset.u8()); Assert.assertEquals(3, reset.u8()); Assert.assertEquals(5678, reset.i64()); @@ -255,7 +255,7 @@ public void testDuplicateRollbackResetOmissionAndRetainedTargetScale() throws Ex binding.decimalColumn("c", new Decimal256(0, 0, 0, 25, 2)); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 2, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); decimalDefinition(reader, "a", QwpConstants.TYPE_DECIMAL64); decimalDefinition(reader, "c", QwpConstants.TYPE_DECIMAL64); Assert.assertEquals(1, reader.u8()); @@ -272,7 +272,7 @@ public void testDuplicateRollbackResetOmissionAndRetainedTargetScale() throws Ex binding.decimalColumn("c", new Decimal64(3, 0)); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableHeader(encoder, resetSize, 1, 2); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); decimalDefinition(reset, "a", QwpConstants.TYPE_DECIMAL64); decimalDefinition(reset, "c", QwpConstants.TYPE_DECIMAL64); Assert.assertEquals(1, reset.u8()); @@ -302,7 +302,7 @@ public void testMissingInfersNativeDecimalWidthAndScale() throws Exception { append(binding, values[i]); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = unknownTableReader(encoder, size, types[i]); + QwpTestWireReader reader = unknownTableReader(encoder, size, types[i]); Assert.assertEquals(0, reader.u8()); Assert.assertEquals(scale(values[i]), reader.u8()); for (long limb : wireLimbs(values[i])) { @@ -355,7 +355,7 @@ public void testLegacyBufferStillChoosesScaleFromFirstValueInEachBatch() throws column.addDecimal64(new Decimal64(12, 2)); buffer.nextRow(); int firstSize = encoder.encode(buffer); - Reader first = legacyTableReader(encoder, firstSize); + QwpTestWireReader first = legacyTableReader(encoder, firstSize); Assert.assertEquals(0, first.u8()); Assert.assertEquals(2, first.u8()); Assert.assertEquals(12, first.i64()); @@ -365,7 +365,7 @@ public void testLegacyBufferStillChoosesScaleFromFirstValueInEachBatch() throws column.addDecimal64(new Decimal64(34, 3)); buffer.nextRow(); int secondSize = encoder.encode(buffer); - Reader second = legacyTableReader(encoder, secondSize); + QwpTestWireReader second = legacyTableReader(encoder, secondSize); Assert.assertEquals(0, second.u8()); Assert.assertEquals(3, second.u8()); Assert.assertEquals(34, second.i64()); @@ -385,7 +385,7 @@ public void testLegacyNullOnlyDecimalUsesWireScaleZeroWithoutLockingNextBatch() buffer.nextRow(); int nullSize = encoder.encode(buffer); - Reader nullOnly = legacyTableReader(encoder, nullSize, QwpConstants.TYPE_DECIMAL256); + QwpTestWireReader nullOnly = legacyTableReader(encoder, nullSize, QwpConstants.TYPE_DECIMAL256); Assert.assertEquals(1, nullOnly.u8()); Assert.assertEquals(1, nullOnly.u8()); Assert.assertEquals(0, nullOnly.u8()); @@ -395,7 +395,7 @@ public void testLegacyNullOnlyDecimalUsesWireScaleZeroWithoutLockingNextBatch() column.addDecimal256(new Decimal256(0, 0, 0, 1234, 4)); buffer.nextRow(); int valueSize = encoder.encode(buffer); - Reader value = legacyTableReader(encoder, valueSize, QwpConstants.TYPE_DECIMAL256); + QwpTestWireReader value = legacyTableReader(encoder, valueSize, QwpConstants.TYPE_DECIMAL256); Assert.assertEquals(0, value.u8()); Assert.assertEquals(4, value.u8()); Assert.assertEquals(1234, value.i64()); @@ -417,7 +417,7 @@ private static void append(QwpSchemaBinding binding, Object value) { } } - private static void assertExpectedLimbs(Reader reader, String[] f) { + private static void assertExpectedLimbs(QwpTestWireReader reader, String[] f) { int count = wireLongCount(f[7]); for (int i = 0; i < count; i++) { Assert.assertEquals(parseHex(f[11 + i]), reader.i64()); @@ -427,10 +427,6 @@ private static void assertExpectedLimbs(Reader reader, String[] f) { } } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - private static void assertTargetTag(String name, int precision) { String expected; if (precision <= 2) { @@ -449,16 +445,6 @@ private static void assertTargetTag(String name, int precision) { Assert.assertEquals(expected, name); } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - private static Object decimal(String[] f) { int scale = Integer.parseInt(f[2]); long ll = parseHex(f[3]); @@ -476,26 +462,11 @@ private static Object decimal(String[] f) { } } - private static void decimalDefinition(Reader reader, String name, byte type) { + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { Assert.assertEquals(name, reader.string()); Assert.assertEquals(type, reader.u8()); } - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - private static long[] limbs(Object value) { if (value instanceof Decimal64) { return new long[]{((Decimal64) value).getValue()}; @@ -507,31 +478,12 @@ private static long[] limbs(Object value) { return new long[]{decimal.getHh(), decimal.getHl(), decimal.getLh(), decimal.getLl()}; } - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static Reader legacyTableReader(QwpWebSocketEncoder encoder, int size) { + private static QwpTestWireReader legacyTableReader(QwpWebSocketEncoder encoder, int size) { return legacyTableReader(encoder, size, QwpConstants.TYPE_DECIMAL64); } - private static Reader legacyTableReader(QwpWebSocketEncoder encoder, int size, byte type) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader legacyTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.varint()); @@ -545,12 +497,6 @@ private static long parseHex(String value) { return Long.parseUnsignedLong(value, 16); } - private static byte[] parameterizedColumn(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6 + 4).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 4).putInt(1).array(); - } - private static int scale(Object value) { if (value instanceof Decimal64) { return ((Decimal64) value).getScale(); @@ -561,30 +507,18 @@ private static int scale(Object value) { return ((Decimal256) value).getScale(); } - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - reader.skip(QwpConstants.HEADER_SIZE); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { - Reader reader = tableHeader(encoder, size, 1, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); decimalDefinition(reader, "value", type); return reader; } - private static Reader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + private static QwpTestWireReader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type) { return unknownTableReader(encoder, size, type, 1); } - private static Reader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(0, reader.u8()); @@ -635,64 +569,4 @@ private static long[] wireLimbs(Object value) { Decimal256 decimal = (Decimal256) value; return new long[]{decimal.getLl(), decimal.getLh(), decimal.getHl(), decimal.getHh()}; } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long i64() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java index 027d592dd..7caeaed22 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java @@ -10,21 +10,20 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingStringDateTest { @@ -58,7 +57,7 @@ public void testCorpusUsesExactDateWire() throws Exception { binding.stringColumn("value", value); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, "value", QwpConstants.TYPE_DATE); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", QwpConstants.TYPE_DATE); if ("NULL".equals(fields[3])) { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); @@ -99,7 +98,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 4, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 2); Assert.assertEquals("a", reader.string()); Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); Assert.assertEquals("c", reader.string()); @@ -116,7 +115,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { binding.stringColumn("a", "1970-01-01 00:00:00.001UTC"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableHeader(encoder, resetSize, 1, 2); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); Assert.assertEquals("a", reset.string()); Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); Assert.assertEquals("c", reset.string()); @@ -137,7 +136,7 @@ public void testNativeInferenceParameterizedAndDesignatedBehavior() { binding.stringColumn("value", "1970-01-01"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(0, reader.u8()); @@ -158,55 +157,6 @@ public void testNativeInferenceParameterizedAndDesignatedBehavior() { } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type) - .putShort((short) params.length).put(params).array(); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - private static CharSequence input(String kind, String value) { Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); if ("".equals(value)) { @@ -226,87 +176,10 @@ private static CharSequence input(String kind, String value) { return new String(chars); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { - Reader reader = tableHeader(encoder, size, rows, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); Assert.assertEquals(name, reader.string()); Assert.assertEquals(type, reader.u8()); return reader; } - - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - reader.skip(QwpConstants.HEADER_SIZE); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position <= limit - Integer.BYTES); - int value = Unsafe.getUnsafe().getInt(address + position); - position += Integer.BYTES; - return value; - } - - private long i64() { - Assert.assertTrue(position <= limit - Long.BYTES); - long value = Unsafe.getUnsafe().getLong(address + position); - position += Long.BYTES; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - while (true) { - int value = u8(); - result |= (value & 0x7f) << shift; - if ((value & 0x80) == 0) { - return result; - } - shift += 7; - } - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java index 4a6599064..ce9ebd77c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java @@ -10,10 +10,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; @@ -21,10 +18,12 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingStringDecimalTest { @@ -63,7 +62,7 @@ public void testCorpusUsesExactTargetDecimalWire() throws Exception { Assert.assertEquals(QwpConstants.FLAG_SCHEMA, Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); - Reader reader = tableReader(encoder, size, wireType(vector.targetType)); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType)); if (vector.isNull()) { Assert.assertEquals(vector.caseId, 1, reader.u8()); Assert.assertEquals(vector.caseId, 1, reader.u8()); @@ -115,7 +114,7 @@ public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 4, 1); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 1); decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(0x0c, reader.u8()); @@ -127,7 +126,7 @@ public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { buffer.reset(); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableHeader(encoder, resetSize, 1, 1); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 1); decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); Assert.assertEquals(1, reset.u8()); Assert.assertEquals(1, reset.u8()); @@ -137,56 +136,11 @@ public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { }); } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(error.getMessage(), reason, error.getReason()); - Assert.assertFalse(error.getMessage(), error.isRetryable()); - } - - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static void decimalDefinition(Reader reader, String name, byte type) { + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { Assert.assertEquals(name, reader.string()); Assert.assertEquals(type, reader.u8()); } - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static QwpSchemaResponse known(byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) -1).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - private static long parseHex(String value) { Assert.assertNotEquals("-", value); return Long.parseUnsignedLong(value, 16); @@ -203,20 +157,8 @@ private static int targetIndex(String target) { } } - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - reader.skip(QwpConstants.HEADER_SIZE); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { - Reader reader = tableHeader(encoder, size, 1, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); decimalDefinition(reader, "value", type); return reader; } @@ -233,64 +175,6 @@ private static int wireLongCount(String target) { return 1; } - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long i64() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) bytes[i] = (byte) u8(); - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } - private static final class Vector { private final String caseId; private final long[] expectedLimbs = new long[4]; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java index d166a1c01..ccc9ac4a3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java @@ -10,21 +10,21 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingStringGeoHashTest { @@ -66,7 +66,7 @@ public void testCorpusUsesExactGeoHashWire() throws Exception { binding.stringColumn("value", value); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); if ("NULL".equals(fields[3])) { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); @@ -80,7 +80,7 @@ public void testCorpusUsesExactGeoHashWire() throws Exception { Assert.assertEquals((int) ((expected >>> (8 * i)) & 0xff), reader.u8()); } } - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); } else { throw new AssertionError("unknown outcome: " + fields[3]); } @@ -117,20 +117,20 @@ public void testDuplicateBitmapRollbackResetInferenceAndGuards() throws Exceptio binding.stringColumn("c", "04"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 5, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 5, 2); Assert.assertEquals("a", reader.string()); Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); Assert.assertEquals("c", reader.string()); Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); assertGeo(reader, 29, 8, 0xff); assertGeo(reader, 15, 8, 1); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); buffer.reset(); binding.stringColumn("a", "04"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableHeader(encoder, resetSize, 1, 2); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); Assert.assertEquals("a", reset.string()); Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); Assert.assertEquals("c", reset.string()); @@ -139,7 +139,7 @@ public void testDuplicateBitmapRollbackResetInferenceAndGuards() throws Exceptio Assert.assertEquals(1, reset.u8()); Assert.assertEquals(1, reset.u8()); Assert.assertEquals(8, reset.varint()); - Assert.assertEquals(resetSize, reset.position); + Assert.assertEquals(resetSize, reset.position()); } try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); @@ -206,9 +206,9 @@ public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exc binding.stringColumn("value", input); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); assertGeo(reader, 0, bits, max); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); buffer.reset(); binding.stringColumn("value", input); @@ -220,7 +220,7 @@ public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exc size = encoder.encodeSchema(buffer); reader = tableReader(encoder, size, 3); assertGeo(reader, 6, bits, max); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); } } @@ -229,7 +229,7 @@ public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exc .addGeoHash(0x0fffffffffffffffL, 60); buffer.nextRow(); int size = encoder.encode(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); Assert.assertEquals(QwpConstants.VERSION, reader.u8()); Assert.assertEquals(QwpConstants.FLAG_GORILLA, reader.u8()); @@ -243,7 +243,7 @@ public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exc Assert.assertEquals(0, reader.u8()); Assert.assertEquals(60, reader.varint()); Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); } try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { @@ -251,16 +251,16 @@ public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exc .addGeoHash(0x0fffffffffffffffL, 60); buffer.nextRow(); int size = encoder.encodeSchema(buffer, 1, 1); - Reader reader = tableReader(encoder, size, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); Assert.assertEquals(0, reader.u8()); Assert.assertEquals(60, reader.varint()); Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); } }); } - private static void assertGeo(Reader reader, int bitmap, int bits, long value) { + private static void assertGeo(QwpTestWireReader reader, int bitmap, int bits, long value) { Assert.assertEquals(bitmap < 0 ? 0 : 1, reader.u8()); if (bitmap >= 0) { Assert.assertEquals(bitmap, reader.u8()); @@ -271,95 +271,10 @@ private static void assertGeo(Reader reader, int bitmap, int bits, long value) { } } - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] parameterizedColumn(String name, int type) { - return column(name, type, new byte[]{1}); - } - - private static byte[] column(String name, int type, byte[] parameters) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6 + parameters.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) parameters.length) - .put(parameters).array(); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) - .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { - Reader reader = tableHeader(encoder, size, rows, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); Assert.assertEquals("value", reader.string()); Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); return reader; } - - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); - Assert.assertEquals(QwpConstants.VERSION, reader.u8()); - Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); - Assert.assertEquals(1, reader.u16()); - Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { this.address = address; this.limit = limit; } - private int i32() { check(4); int value = Unsafe.getUnsafe().getInt(address + position); position += 4; return value; } - private long i64() { check(8); long value = Unsafe.getUnsafe().getLong(address + position); position += 8; return value; } - private void skip(int length) { check(length); position += length; } - private String string() { int length = varint(); byte[] bytes = new byte[length]; for (int i = 0; i < length; i++) bytes[i] = (byte) u8(); return new String(bytes, StandardCharsets.UTF_8); } - private int u8() { check(1); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } - private int u16() { check(2); int value = Unsafe.getUnsafe().getShort(address + position) & 0xffff; position += 2; return value; } - private int varint() { int result = 0; int shift = 0; int b; do { b = u8(); result |= (b & 0x7f) << shift; shift += 7; } while ((b & 0x80) != 0); return result; } - private void check(int size) { Assert.assertTrue(position + size <= limit); } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java index a34216733..a1023ca4a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java @@ -10,21 +10,21 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingStringLong256Test { @@ -59,7 +59,7 @@ public void testCorpusUsesExactLong256Wire() throws Exception { binding.stringColumn("value", fields[1]); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); Assert.assertEquals(0, reader.u8()); for (int i = 3; i < 7; i++) { Assert.assertEquals(parseHexLong(fields[i]), reader.i64()); @@ -70,7 +70,7 @@ public void testCorpusUsesExactLong256Wire() throws Exception { binding.stringColumn("value", null); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(size, reader.position()); @@ -114,7 +114,7 @@ public void testDuplicateNullOmissionRollbackAndReset() throws Exception { binding.stringColumn("c", "0x04"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.u8()); @@ -134,7 +134,7 @@ public void testDuplicateNullOmissionRollbackAndReset() throws Exception { binding.stringColumn("a", "0x05"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); reset.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reset.string()); Assert.assertEquals(1, reset.u8()); @@ -182,13 +182,13 @@ public void testParameterizedDesignatedAndOtherTargetsRemainUnsupported() { } try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { QwpSchemaBinding binding = new QwpSchemaBinding(buffer, - response(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("ts", "0x00")); } } - private static void assertLong256Values(Reader reader, int bitmap, long value) { + private static void assertLong256Values(QwpTestWireReader reader, int bitmap, long value) { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(bitmap, reader.u8()); Assert.assertEquals(value, reader.i64()); @@ -197,64 +197,12 @@ private static void assertLong256Values(Reader reader, int bitmap, long value) { Assert.assertEquals(0, reader.i64()); } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return new QwpSchemaBinding(buffer, response(columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] parameterizedColumn(String name, int type) { - return column(name, type, new byte[]{1}); - } - - private static byte[] column(String name, int type, byte[] parameters) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6 + parameters.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) parameters.length) - .put(parameters).array(); - } - - private static QwpSchemaResponse missing() { - ByteBuffer payload = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING); - return decode(payload.array()); - } - - private static QwpSchemaResponse response(byte[]... columns) { - return response(-1, columns); - } - - private static QwpSchemaResponse response(int designatedIndex, byte[]... columns) { - int length = 26; - for (byte[] column : columns) length += column.length; - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); - for (byte[] column : columns) payload.put(column); - return decode(payload.array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { return tableReader(encoder, size, rows, "value"); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.u8()); @@ -270,40 +218,4 @@ private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int row private static long parseHexLong(String value) { return Long.parseUnsignedLong(value, 16); } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(reason, error.getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private String ascii(int length) { return new String(bytes(length), StandardCharsets.UTF_8); } - private byte[] bytes(int length) { - byte[] value = new byte[length]; - for (int i = 0; i < length; i++) value[i] = (byte) u8(); - return value; - } - private int i32() { int value = Unsafe.getUnsafe().getInt(address + position); position += 4; return value; } - private long i64() { long value = Unsafe.getUnsafe().getLong(address + position); position += 8; return value; } - private int position() { return position; } - private void skip(int length) { position += length; Assert.assertTrue(position <= limit); } - private String string() { return ascii(varint()); } - private int u8() { Assert.assertTrue(position < limit); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } - private int varint() { - int result = 0; - int shift = 0; - int value; - do { value = u8(); result |= (value & 0x7f) << shift; shift += 7; } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java index 51747d5b5..754464584 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java @@ -10,22 +10,20 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.time.temporal.ChronoUnit; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingStringNumericTest { @@ -54,12 +52,12 @@ public void testConformanceCorpusUsesExactTargetWire() throws Exception { } else { binding.stringColumn("value", value); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[3]), 1, 1); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[3]), 1, 1); if ("".equals(f[4])) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); } else { - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); byte[] expected = littleEndianHex(f[4]); Assert.assertArrayEquals(f[0], expected, reader.bytes(expected.length)); } @@ -85,18 +83,18 @@ public void testDuplicatePrecedesInvalidParseAndUnsupportedConversion() throws E binding.uuidColumn("uuid", 1, 2).stringColumn("uuid", "not-a-uuid"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, 1, 2); Assert.assertEquals("n", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_INT, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); Assert.assertEquals("uuid", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_UUID, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(42, reader.intValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(2, reader.longValue()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(42, reader.i32()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); Assert.assertEquals(size, reader.position()); } try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { @@ -104,10 +102,10 @@ public void testDuplicatePrecedesInvalidParseAndUnsupportedConversion() throws E binding.timestampColumn("value", 123, ChronoUnit.MICROS) .stringColumn("value", "not-a-timestamp-conversion"); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_TIMESTAMP, 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(123, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_TIMESTAMP, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(123, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -129,19 +127,19 @@ public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws binding.stringColumn("c", "-0.0"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, 2, 2); Assert.assertEquals("a", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_INT, reader.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); Assert.assertEquals("c", reader.string()); - Assert.assertEquals(QwpConstants.TYPE_DOUBLE, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0x8000000000000000L, reader.longValue()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x8000000000000000L, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -156,17 +154,17 @@ public void testResetClearAndMutableInput() throws Exception { binding.stringColumn("value", mutable); mutable.replace(0, 1, "9"); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(7, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); buffer.reset(); binding.stringColumn("value", "8"); buffer.nextRow(); reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(8, reader.longValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(8, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); buffer.clear(); assertIllegalState(() -> binding.stringColumn("value", "1")); @@ -183,10 +181,10 @@ public void testNullAndParsedSentinelsHaveDistinctWirePresence() throws Exceptio buffer.nextRow(); binding.stringColumn("value", null); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_INT, 2, 1); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(Integer.MIN_VALUE, reader.intValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_INT, 2, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(Integer.MIN_VALUE, reader.i32()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -217,59 +215,15 @@ public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); - } - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1) - .put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + length).order(ByteOrder.LITTLE_ENDIAN).putInt(QwpConstants.MAGIC_MESSAGE) - .put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(length).put(payload.array()); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTablePrefix(reader, rows, columns); Assert.assertEquals("value", reader.string()); - Assert.assertEquals(type, reader.byteValue()); + Assert.assertEquals(type, reader.u8()); return reader; } - private static void skipTablePrefix(Reader reader, int rows, int columns) { - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - } - private static int targetType(String value) { switch (value) { case "BYTE": return ColumnType.BYTE; @@ -321,12 +275,6 @@ private static void assertIllegalState(Runnable runnable) { Assert.assertThrows(IllegalStateException.class, runnable::run); } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable runnable) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); - Assert.assertEquals(reason, error.getReason()); - Assert.assertFalse(error.isRetryable()); - } - private static void assertSchemaError( LineSenderSchemaException.Reason reason, String column, @@ -341,68 +289,4 @@ private static void assertSchemaError( Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=STRING")); Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + target)); } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int byteValue() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 255; - } - - private byte[] bytes(int count) { - Assert.assertTrue(position + count <= limit); - byte[] bytes = new byte[count]; - for (int i = 0; i < count; i++) { - bytes[i] = (byte) byteValue(); - } - return bytes; - } - - private int intValue() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long longValue() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int count) { - Assert.assertTrue(position + count <= limit); - position += count; - } - - private String string() { - return new String(bytes(varint()), StandardCharsets.UTF_8); - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = byteValue(); - result |= (value & 127) << shift; - shift += 7; - } while ((value & 128) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java index 6d2428e8d..701fbea43 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java @@ -10,21 +10,20 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingStringTimestampTest { @@ -62,7 +61,7 @@ public void testCorpusUsesExactTargetUnitWire() throws Exception { binding.stringColumn("value", value); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, "value", wireType); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", wireType); if ("NULL".equals(fields[4])) { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(1, reader.u8()); @@ -99,7 +98,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { buffer.nextRow(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 3, "value", wireType); + QwpTestWireReader reader = tableReader(encoder, size, 3, "value", wireType); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(5, reader.u8()); Assert.assertEquals(0, reader.u8()); @@ -110,7 +109,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { binding.stringColumn("value", "1969-12-31T23:59:59.999999Z"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableReader(encoder, resetSize, 1, "value", wireType); + QwpTestWireReader reset = tableReader(encoder, resetSize, 1, "value", wireType); Assert.assertEquals(0, reset.u8()); Assert.assertEquals(0, reset.u8()); Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000, reset.i64()); @@ -134,7 +133,7 @@ public void testNullDuplicateOmissionRollbackAndReset() throws Exception { binding.stringColumn("c", "1970-01-01T00:00:00.000004Z"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 2, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); Assert.assertEquals("a", reader.string()); Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); Assert.assertEquals("c", reader.string()); @@ -159,7 +158,7 @@ public void testNativeInferenceAndNamedDesignatedRejection() { binding.stringColumn("value", "1970-01-01T00:00:00Z"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(0, reader.u8()); @@ -180,69 +179,13 @@ public void testNativeInferenceAndNamedDesignatedRejection() { } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { - Reader reader = tableHeader(encoder, size, rows, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); Assert.assertEquals(name, reader.string()); Assert.assertEquals(type, reader.u8()); return reader; } - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - reader.skip(QwpConstants.HEADER_SIZE); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - private static int targetType(String value) { Assert.assertTrue(value, "TIMESTAMP".equals(value) || "TIMESTAMP_NANOS".equals(value)); return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; @@ -267,75 +210,4 @@ private static CharSequence input(String kind, String value) { private static byte wireType(String value) { return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int i32() { - Assert.assertTrue(position <= limit - Integer.BYTES); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private String ascii(int length) { - byte[] value = new byte[length]; - for (int i = 0; i < length; i++) { - value[i] = (byte) u8(); - } - return new String(value, StandardCharsets.UTF_8); - } - - private long i64() { - Assert.assertTrue(position <= limit - Long.BYTES); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index 0401a9937..ee7e15a40 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -33,7 +33,6 @@ import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Numbers; import io.questdb.client.std.Unsafe; import org.junit.Assert; @@ -42,12 +41,14 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.temporal.ChronoUnit; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.schemaResult; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingTest { @@ -89,10 +90,10 @@ public void testTimestampInputConformanceCorpus() throws Exception { } write.run(); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); - Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.byteValue()); - Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.byteValue()); - Assert.assertEquals(Long.parseLong(fields[5]), reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.u8()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } } catch (AssertionError e) { @@ -112,11 +113,11 @@ public void testTimestampOverloadValidationAndDuplicateSuppression() throws Exce .timestampColumn("ts", (Instant) null) .timestampColumn("ts", Long.MAX_VALUE, ChronoUnit.WEEKS); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, QwpConstants.TYPE_TIMESTAMP_NANOS); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(123, reader.longValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(123, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { @@ -156,16 +157,16 @@ public void testTimestampInstantFailureRollsBackOnlyCurrentRowColumns() throws E rows.timestampColumn("micro", Instant.ofEpochSecond(3)) .timestampColumn("late", Instant.ofEpochSecond(0, 4)); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1_000_000, reader.longValue()); - Assert.assertEquals(3_000_000, reader.longValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(4, reader.longValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1_000_000, reader.i64()); + Assert.assertEquals(3_000_000, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -199,10 +200,10 @@ public void testTimestampUnitConformanceCorpus() throws Exception { } rows.timestampColumn("ts", input, unit); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); - Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.byteValue()); - Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.byteValue()); - Assert.assertEquals(Long.parseLong(fields[4]), reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.u8()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } } catch (AssertionError e) { @@ -234,16 +235,16 @@ public void testTimestampValidationDuplicateAndRollback() throws Exception { rows.timestampColumn("micro", 9, ChronoUnit.MICROS) .timestampColumn("nano", 10, ChronoUnit.NANOS); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(7, reader.longValue()); - Assert.assertEquals(9, reader.longValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(10, reader.longValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(9, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(10, reader.i64()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -261,14 +262,14 @@ public void testTimestampNullUnsupportedAndBindingLifecycle() throws Exception { buffer.nextRow(); rows.stringColumn("micro", null); buffer.nextRow(); - Reader nulls = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpTestWireReader nulls = tableReader(encoder, encoder.encodeSchema(buffer), 2, QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); - Assert.assertEquals(1, nulls.byteValue()); - Assert.assertEquals(3, nulls.byteValue()); - Assert.assertEquals(0, nulls.byteValue()); - Assert.assertEquals(1, nulls.byteValue()); - Assert.assertEquals(3, nulls.byteValue()); - Assert.assertEquals(0, nulls.byteValue()); + Assert.assertEquals(1, nulls.u8()); + Assert.assertEquals(3, nulls.u8()); + Assert.assertEquals(0, nulls.u8()); + Assert.assertEquals(1, nulls.u8()); + Assert.assertEquals(3, nulls.u8()); + Assert.assertEquals(0, nulls.u8()); Assert.assertEquals(encoder.getBuffer().getPosition(), nulls.position()); buffer.reset(); @@ -324,7 +325,7 @@ public void testClassifiesSchemaLessResponses() throws Exception { assertCtorReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "access denied"); assertCtorReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "temporarily unavailable"); try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { - new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)).longColumn("x", 1); + new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)).longColumn("x", 1); } assertCtorReason(QwpSchemaProtocol.RESULT_TOO_LARGE, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "response limit"); } @@ -333,7 +334,7 @@ public void testClassifiesSchemaLessResponses() throws Exception { public void testInfersNativeTargetsForMissingSchemaAndColumns() throws Exception { assertMemoryLeak(() -> { try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.longColumn("l", 1) .stringColumn("s", "x") .symbol("sym", "y") @@ -361,22 +362,22 @@ public void testInfersNativeTargetsForMissingSchemaAndColumns() throws Exception try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.uuidColumn("u", 0x0102030405060708L, 0x1112131415161718L) .floatColumn("f", Float.intBitsToFloat(0x3fc00000)) .stringColumn("s", "quest"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, -1, -1, 1, + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_UUID, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_VARCHAR); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0x0102030405060708L, reader.longValue()); - Assert.assertEquals(0x1112131415161718L, reader.longValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0x3fc00000, reader.intValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(5, reader.intValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x3fc00000, reader.i32()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(5, reader.i32()); Assert.assertEquals("quest", reader.ascii(5)); Assert.assertEquals(size, reader.position()); } @@ -398,7 +399,7 @@ public void testInfersNativeTargetsForMissingSchemaAndColumns() throws Exception try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.longColumn("a", 1); buffer.nextRow(); try { @@ -430,15 +431,15 @@ public CharSequence subSequence(int start, int end) { rows.longColumn("a", 3); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, -1, -1, 2, QwpConstants.TYPE_LONG); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(3, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); Assert.assertEquals(size, reader.position()); } try (QwpTableBuffer micros = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(micros, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(micros, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.designatedTimestamp(1, ChronoUnit.MICROS); Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, micros.getColumnDefs()[0].getTypeCode()); micros.nextRow(); @@ -448,7 +449,7 @@ public CharSequence subSequence(int start, int end) { micros.rollbackUncommittedColumns(); } try (QwpTableBuffer nanos = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(nanos, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(nanos, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.designatedTimestamp(1, ChronoUnit.NANOS); Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, nanos.getColumnDefs()[0].getTypeCode()); } @@ -468,15 +469,15 @@ public void testDuplicateUsesFirstValueAcrossInputSetters() throws Exception { .stringColumn("u", "malformed"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(2, reader.longValue()); - Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); - Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(4, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(4, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -548,12 +549,12 @@ public void testSmallIntegerNumericConformanceCorpus() throws Exception { } else { append.run(); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, numericWireType(fields[3])); if ("".equals(fields[4])) { assertNumericNull(reader, fields[3]); } else { - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); assertNumericValue(reader, fields[3], fields[4]); } Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); @@ -590,29 +591,29 @@ public void testSmallIntegerTextConformanceCorpusUsesTargetWire() throws Excepti QwpSchemaBinding rows = rows(buffer, column("value", textColumnType(fields[3]))); appendSmallInteger(rows, fields[1], input, "value"); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); if ("".equals(fields[4])) { Assert.assertEquals("", fields[5]); - Assert.assertEquals("NULL bitmap must be present", 1, reader.byteValue()); - Assert.assertEquals("row zero must be NULL", 1, reader.byteValue()); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("row zero must be NULL", 1, reader.u8()); if (wireType == QwpConstants.TYPE_SYMBOL) { Assert.assertEquals(0, reader.varint()); } else { - Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(0, reader.i32()); } } else { byte[] expected = hexBytes(fields[4]); Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); - Assert.assertEquals("value must be bitmap-present", 0, reader.byteValue()); + Assert.assertEquals("value must be bitmap-present", 0, reader.u8()); if (wireType == QwpConstants.TYPE_SYMBOL) { Assert.assertEquals(1, reader.varint()); - Assert.assertEquals(fields[5], reader.stringValue()); + Assert.assertEquals(fields[5], reader.string()); Assert.assertEquals(0, reader.varint()); } else { - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); for (byte value : expected) { - Assert.assertEquals(value & 0xff, reader.byteValue()); + Assert.assertEquals(value & 0xff, reader.u8()); } } } @@ -664,7 +665,7 @@ public void testSmallIntegerDecimalConformanceCorpusUsesTargetWire() throws Exce append.run(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader( + QwpTestWireReader reader = tableReader( encoder, size, 1, @@ -672,17 +673,17 @@ public void testSmallIntegerDecimalConformanceCorpusUsesTargetWire() throws Exce ); if ("NULL".equals(fields[6])) { Assert.assertEquals("NULL", fields[11]); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); } else { Assert.assertEquals("VALUE", fields[6]); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); } - Assert.assertEquals(scale, reader.byteValue()); + Assert.assertEquals(scale, reader.u8()); if ("VALUE".equals(fields[6])) { int limbs = decimalWireLongCount(fields[3]); for (int i = 0; i < limbs; i++) { - Assert.assertEquals(parseHexLong(fields[7 + i]), reader.longValue()); + Assert.assertEquals(parseHexLong(fields[7 + i]), reader.i64()); } for (int i = limbs; i < 4; i++) { Assert.assertEquals("-", fields[7 + i]); @@ -728,23 +729,23 @@ public void testIntegerTemporalConformanceCorpusUsesTargetWire() throws Exceptio QwpSchemaBinding rows = rows(buffer, column("value", targetType)); appendIntegerTemporal(rows, fields[1], input, "value"); buffer.nextRow(); - Reader reader = tableReader( + QwpTestWireReader reader = tableReader( encoder, encoder.encodeSchema(buffer), 1, temporalWireType(fields[3]) ); if ("".equals(fields[4])) { - Assert.assertEquals("NULL bitmap must be present", 1, reader.byteValue()); - Assert.assertEquals("row zero must be NULL", 1, reader.byteValue()); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("row zero must be NULL", 1, reader.u8()); } else { - Assert.assertEquals("value must be bitmap-present", 0, reader.byteValue()); + Assert.assertEquals("value must be bitmap-present", 0, reader.u8()); } if (!"DATE".equals(fields[3])) { - Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.byteValue()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); } if (!"".equals(fields[4])) { - Assert.assertEquals(Long.parseLong(fields[4]), reader.longValue()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); } Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } @@ -778,14 +779,14 @@ public void testSmallIntegerDuplicateFirstAndFailureRollback() throws Exception rows.intColumn("c", 3); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 2, + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG, QwpConstants.TYPE_DOUBLE); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(-1, reader.longValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(Double.doubleToRawLongBits(3), reader.longValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(Double.doubleToRawLongBits(3), reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -804,9 +805,9 @@ public void testIntMinIsNullWithCompanionMissingRowsAcrossNumericTargets() throw rows.intColumn("n", 7); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 3, numericWireType(target)); - Assert.assertEquals(target, 1, reader.byteValue()); - Assert.assertEquals(target, 3, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, 3, numericWireType(target)); + Assert.assertEquals(target, 1, reader.u8()); + Assert.assertEquals(target, 3, reader.u8()); if ("FLOAT".equals(target)) { assertNumericValue(reader, target, "0x40e00000"); } else if ("DOUBLE".equals(target)) { @@ -841,13 +842,13 @@ public void testIntToIpv4UsesTargetNativeWireAndNulls() throws Exception { buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 7, QwpConstants.TYPE_IPv4); - Assert.assertEquals("NULL bitmap must be present", 1, reader.byteValue()); - Assert.assertEquals("INT null, IPv4 zero and omission must be NULL", 0x43, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(Integer.MIN_VALUE + 1, reader.intValue()); - Assert.assertEquals(Integer.MAX_VALUE, reader.intValue()); - Assert.assertEquals(-1, reader.intValue()); + QwpTestWireReader reader = tableReader(encoder, size, 7, QwpConstants.TYPE_IPv4); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("INT null, IPv4 zero and omission must be NULL", 0x43, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(Integer.MIN_VALUE + 1, reader.i32()); + Assert.assertEquals(Integer.MAX_VALUE, reader.i32()); + Assert.assertEquals(-1, reader.i32()); Assert.assertEquals(size, reader.position()); } }); @@ -858,18 +859,18 @@ public void testSmallIntegerMissingSchemaUsesNativeWireTypes() throws Exception assertMemoryLeak(() -> { try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.byteColumn("b", (byte) -1).shortColumn("s", (short) 2).intColumn("i", 3); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, -1, -1, 1, + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, QwpConstants.TYPE_INT); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0xff, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(2, reader.shortValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(3, reader.intValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0xff, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(2, reader.u16()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(3, reader.i32()); Assert.assertEquals(size, reader.position()); } }); @@ -956,16 +957,16 @@ public void testLongNumericConformanceCorpus() throws Exception { rows.longColumn("n", input); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, wireType); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); if ("".equals(fields[3])) { if (input == Long.MIN_VALUE) { assertNumericNull(reader, fields[2]); } else { - Assert.assertEquals("INT sentinel is carried as a non-null wire value", 0, reader.byteValue()); - Assert.assertEquals(Integer.MIN_VALUE, reader.intValue()); + Assert.assertEquals("INT sentinel is carried as a non-null wire value", 0, reader.u8()); + Assert.assertEquals(Integer.MIN_VALUE, reader.i32()); } } else { - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); assertNumericValue(reader, fields[2], fields[3]); } Assert.assertEquals(size, reader.position()); @@ -993,13 +994,13 @@ public void testLongNumericDuplicateFirstAndFailureRollback() throws Exception { rows.longColumn("b", 3).longColumn("i", 4); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_INT); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(7, reader.byteValue()); - Assert.assertEquals(3, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(4, reader.intValue()); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(4, reader.i32()); Assert.assertEquals(size, reader.position()); } }); @@ -1036,10 +1037,10 @@ public void testLongNullRejectsUuidUnlessDuplicate() throws Exception { rows.uuidColumn("u", 1, 2).longColumn("u", Long.MIN_VALUE); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(2, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -1062,14 +1063,14 @@ public void testLongNumericDuplicateFirstAcrossEveryTarget() throws Exception { rows.longColumn("b", 128).longColumn("s", 32768).longColumn("i", 2147483648L); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); - Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.shortValue()); - Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.intValue()); - Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(7, reader.longValue()); - Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(Float.floatToRawIntBits(7), reader.intValue()); - Assert.assertEquals(0, reader.byteValue()); Assert.assertEquals(Double.doubleToRawLongBits(7), reader.longValue()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.u16()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.i32()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Float.floatToRawIntBits(7), reader.i32()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Double.doubleToRawLongBits(7), reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -1091,10 +1092,10 @@ public void testNumericFailureRemovesColumnsIntroducedOnlyInFailedRow() throws E rows.longColumn("a", 3); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(3, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -1125,7 +1126,7 @@ public void testNullStringSupportsAllNumericTargets() throws Exception { .stringColumn("l", null).stringColumn("f", null).stringColumn("d", null); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); assertNumericNull(reader, "BYTE"); assertNumericNull(reader, "SHORT"); @@ -1147,11 +1148,11 @@ public void testNullStringSupportsLongAndUuidTargets() throws Exception { rows.stringColumn("u", null).stringColumn("l", null); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); Assert.assertEquals(size, reader.position()); } }); @@ -1229,14 +1230,14 @@ public void testOwnerRollbackAfterSetterFailurePreservesCompletedRows() throws E rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG, QwpConstants.TYPE_UUID); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(2, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); - Assert.assertEquals(0x0123456789abcdefL, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG, QwpConstants.TYPE_UUID); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -1263,9 +1264,9 @@ public void testAttachmentLifecycleAndStaleBindingGuards() throws Exception { buffer.nextRow(); try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_LONG); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(7, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); Assert.assertEquals(size, reader.position()); } } @@ -1368,12 +1369,12 @@ public void testStringAndTypedUuidEncodeSameLimbs() throws Exception { rows.uuidColumn("u", lo, hi); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(lo, reader.longValue()); - Assert.assertEquals(hi, reader.longValue()); - Assert.assertEquals(lo, reader.longValue()); - Assert.assertEquals(hi, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(lo, reader.i64()); + Assert.assertEquals(hi, reader.i64()); + Assert.assertEquals(lo, reader.i64()); + Assert.assertEquals(hi, reader.i64()); Assert.assertEquals(size, reader.position()); } }); @@ -1407,22 +1408,22 @@ public void testIpv4ConformanceCorpusUsesExactTargetWire() throws Exception { byte wireType = targetType == ColumnType.IPv4 ? QwpConstants.TYPE_IPv4 : QwpConstants.TYPE_VARCHAR; - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); if ("NULL".equals(fields[3])) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); if (wireType == QwpConstants.TYPE_VARCHAR) { - Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(0, reader.i32()); } } else { Assert.assertEquals("VALUE", fields[3]); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); if (wireType == QwpConstants.TYPE_IPv4) { - Assert.assertEquals((int) Long.parseUnsignedLong(fields[4], 16), reader.intValue()); + Assert.assertEquals((int) Long.parseUnsignedLong(fields[4], 16), reader.i32()); } else { byte[] expected = fields[5].getBytes(StandardCharsets.UTF_8); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); Assert.assertEquals(fields[5], reader.ascii(expected.length)); } } @@ -1462,40 +1463,40 @@ public void testIpv4DuplicateRollbackNullOmissionAndInference() throws Exception buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 5, QwpConstants.TYPE_IPv4); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0x0e, reader.byteValue()); - Assert.assertEquals(0x01020304, reader.intValue()); - Assert.assertEquals(0x0d0e0f10, reader.intValue()); + QwpTestWireReader reader = tableReader(encoder, size, 5, QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0e, reader.u8()); + Assert.assertEquals(0x01020304, reader.i32()); + Assert.assertEquals(0x0d0e0f10, reader.i32()); Assert.assertEquals(size, reader.position()); } try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.ipv4Column("ignored", (CharSequence) null) .ipv4Column("ip", "192.168.1.1"); buffer.nextRow(); Assert.assertEquals(1, buffer.getColumnDefs().length); Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, QwpConstants.TYPE_IPv4); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0xc0a80101, reader.intValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0xc0a80101, reader.i32()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.ipv4Column("ip", 0); buffer.nextRow(); Assert.assertEquals(1, buffer.getColumnDefs().length); Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, QwpConstants.TYPE_IPv4); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); } }); @@ -1551,25 +1552,25 @@ public void testLong256ConformanceCorpusUsesExactTargetWire() throws Exception { ? QwpConstants.TYPE_LONG256 : QwpConstants.TYPE_VARCHAR; int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, wireType); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); if ("NULL".equals(fields[5])) { Assert.assertEquals("-", fields[6]); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); if (wireType == QwpConstants.TYPE_VARCHAR) { - Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(0, reader.i32()); } } else { Assert.assertEquals("VALUE", fields[5]); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); if (wireType == QwpConstants.TYPE_LONG256) { - Assert.assertEquals(l0, reader.longValue()); - Assert.assertEquals(l1, reader.longValue()); - Assert.assertEquals(l2, reader.longValue()); - Assert.assertEquals(l3, reader.longValue()); + Assert.assertEquals(l0, reader.i64()); + Assert.assertEquals(l1, reader.i64()); + Assert.assertEquals(l2, reader.i64()); + Assert.assertEquals(l3, reader.i64()); } else { - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(fields[6].length(), reader.intValue()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(fields[6].length(), reader.i32()); Assert.assertEquals(fields[6], reader.ascii(fields[6].length())); } } @@ -1605,27 +1606,27 @@ public void testLong256DuplicateNullOmissionRollbackAndMissingInference() throws buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_LONG256); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0x06, reader.byteValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(2, reader.longValue()); - Assert.assertEquals(3, reader.longValue()); - Assert.assertEquals(4, reader.longValue()); + QwpTestWireReader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x06, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(4, reader.i64()); Assert.assertEquals(size, reader.position()); } try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); buffer.nextRow(); Assert.assertEquals(1, buffer.getColumnDefs().length); Assert.assertEquals(QwpConstants.TYPE_LONG256, buffer.getColumnDefs()[0].getTypeCode()); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_LONG256); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); Assert.assertEquals(size, reader.position()); } }); @@ -1681,18 +1682,18 @@ public void testNativeGeoHashConformanceCorpusUsesExactTargetWire() throws Excep ? QwpConstants.TYPE_GEOHASH : QwpConstants.TYPE_VARCHAR; int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, wireType); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); if (wireType == QwpConstants.TYPE_GEOHASH) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(precision, reader.varint()); for (int i = 0; i < (precision + 7) / 8; i++) { - Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.byteValue()); + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.u8()); } } else { - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(fields[4].length(), reader.intValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(fields[4].length(), reader.i32()); Assert.assertEquals(fields[4], reader.ascii(fields[4].length())); } Assert.assertEquals(size, reader.position()); @@ -1718,12 +1719,12 @@ public void testNativeGeoHashEveryPrecisionUsesFixedWidthText() throws Exception .geoHashColumn("value", -1, precision); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_VARCHAR); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(precision, reader.intValue()); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(precision, reader.i32()); for (int i = 0; i < precision; i++) { - Assert.assertEquals('1', reader.byteValue()); + Assert.assertEquals('1', reader.u8()); } Assert.assertEquals(size, reader.position()); } @@ -1751,13 +1752,13 @@ public void testNativeGeoHashOmissionRollbackInferenceAndGuards() throws Excepti buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_GEOHASH); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0x02, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x02, reader.u8()); Assert.assertEquals(20, reader.varint()); for (long value : new long[]{0xabcde, 0x12345}) { for (int i = 0; i < 3; i++) { - Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), reader.byteValue()); + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), reader.u8()); } } Assert.assertEquals(size, reader.position()); @@ -1765,18 +1766,18 @@ public void testNativeGeoHashOmissionRollbackInferenceAndGuards() throws Excepti try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = new QwpSchemaBinding(buffer, result(QwpSchemaProtocol.RESULT_MISSING)); + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); rows.geoHashColumn("value", 0xabcde, 20); buffer.nextRow(); Assert.assertEquals(QwpConstants.TYPE_GEOHASH, buffer.getColumnDefs()[0].getTypeCode()); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_GEOHASH); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(20, reader.varint()); - Assert.assertEquals(0xde, reader.byteValue()); - Assert.assertEquals(0xbc, reader.byteValue()); - Assert.assertEquals(0x0a, reader.byteValue()); + Assert.assertEquals(0xde, reader.u8()); + Assert.assertEquals(0xbc, reader.u8()); + Assert.assertEquals(0x0a, reader.u8()); Assert.assertEquals(size, reader.position()); } @@ -1819,18 +1820,18 @@ public void testNativeGeoHashTextOverloadPreservesSourcePrecision() throws Excep .geoHashColumn("text", "U"); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1, + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_GEOHASH, QwpConstants.TYPE_VARCHAR); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(20, reader.varint()); long expected = Numbers.parseGeoHashBase32("u33d"); for (int i = 0; i < 3; i++) { - Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.byteValue()); + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.u8()); } - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(5, reader.intValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(5, reader.i32()); Assert.assertEquals("11010", reader.ascii(5)); Assert.assertEquals(size, reader.position()); } @@ -1857,7 +1858,7 @@ private static void assertCtorReason(int result, LineSenderSchemaException.Reaso try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { try { - new QwpSchemaBinding(buffer, result(result)); + new QwpSchemaBinding(buffer, schemaResult(result)); Assert.fail("expected schema exception"); } catch (LineSenderSchemaException e) { Assert.assertEquals(reason, e.getReason()); @@ -1872,10 +1873,10 @@ private static void assertCtorReason(int result, LineSenderSchemaException.Reaso } private static void assertLongFrame(QwpWebSocketEncoder encoder, int size, long... values) { - Reader reader = tableReader(encoder, size, values.length, QwpConstants.TYPE_LONG); - Assert.assertEquals(0, reader.byteValue()); + QwpTestWireReader reader = tableReader(encoder, size, values.length, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); for (long value : values) { - Assert.assertEquals(value, reader.longValue()); + Assert.assertEquals(value, reader.i64()); } Assert.assertEquals(size, reader.position()); } @@ -1990,34 +1991,34 @@ private static byte[] hexBytes(String value) { return bytes; } - private static void assertNumericValue(Reader reader, String target, String expected) { + private static void assertNumericValue(QwpTestWireReader reader, String target, String expected) { switch (target) { case "BYTE": - Assert.assertEquals(Integer.parseInt(expected) & 0xff, reader.byteValue()); + Assert.assertEquals(Integer.parseInt(expected) & 0xff, reader.u8()); break; case "SHORT": - Assert.assertEquals(Integer.parseInt(expected) & 0xffff, reader.shortValue()); + Assert.assertEquals(Integer.parseInt(expected) & 0xffff, reader.u16()); break; case "INT": - Assert.assertEquals(Integer.parseInt(expected), reader.intValue()); + Assert.assertEquals(Integer.parseInt(expected), reader.i32()); break; case "LONG": - Assert.assertEquals(Long.parseLong(expected), reader.longValue()); + Assert.assertEquals(Long.parseLong(expected), reader.i64()); break; case "FLOAT": - Assert.assertEquals((int) Long.parseUnsignedLong(expected.substring(2), 16), reader.intValue()); + Assert.assertEquals((int) Long.parseUnsignedLong(expected.substring(2), 16), reader.i32()); break; case "DOUBLE": - Assert.assertEquals(Long.parseUnsignedLong(expected.substring(2), 16), reader.longValue()); + Assert.assertEquals(Long.parseUnsignedLong(expected.substring(2), 16), reader.i64()); break; default: Assert.fail(target); } } - private static void assertNumericNull(Reader reader, String target) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(target, 1, reader.byteValue()); + private static void assertNumericNull(QwpTestWireReader reader, String target) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(target, 1, reader.u8()); } private static void rollbackCurrentRow(QwpTableBuffer buffer) { @@ -2156,76 +2157,15 @@ private static void assertEncoderUnchangedIllegalArgument(QwpWebSocketEncoder en Assert.assertArrayEquals(before, copyFrame(encoder, position)); } - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - try { - action.run(); - Assert.fail("expected schema exception"); - } catch (LineSenderSchemaException e) { - Assert.assertEquals(reason, e.getReason()); - Assert.assertFalse(e.isRetryable()); - } - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) params.length).put(params).array(); - } - - private static QwpSchemaResponse known(byte[]... columns) { - return known(-1, columns); - } - - private static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(frame(payload.array())); - } - - private static QwpSchemaResponse result(int result) { - return decode(frame(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) result).array())); - } - private static QwpSchemaBinding rows(QwpTableBuffer buffer, byte[]... columns) { return new QwpSchemaBinding(buffer, known(columns)); } - private static byte[] frame(byte[] payload) { - return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload).array(); - } - - private static QwpSchemaResponse decode(byte[] frame) { - long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.length; i++) { - Unsafe.getUnsafe().putByte(address + i, frame[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.length); - } finally { - Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte... wireTypes) { + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte... wireTypes) { return tableReader(encoder, size, 1, 1, rows, wireTypes); } - private static Reader tableReader( + private static QwpTestWireReader tableReader( QwpWebSocketEncoder encoder, int size, int tableId, @@ -2233,96 +2173,26 @@ private static Reader tableReader( int rows, byte... wireTypes ) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.intValue()); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.byteValue()); - Assert.assertEquals(1, reader.shortValue()); - Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.intValue()); - Assert.assertEquals("t", reader.stringValue()); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); if (tableId < 0) { - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); } else { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(tableId, reader.intValue()); - Assert.assertEquals(metadataVersion, reader.longValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(tableId, reader.i32()); + Assert.assertEquals(metadataVersion, reader.i64()); } Assert.assertEquals(rows, reader.varint()); Assert.assertEquals(wireTypes.length, reader.varint()); for (byte wireType : wireTypes) { - reader.stringValue(); - Assert.assertEquals(wireType, reader.byteValue()); + reader.string(); + Assert.assertEquals(wireType, reader.u8()); } return reader; } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int byteValue() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private String ascii(int length) { - StringBuilder sink = new StringBuilder(length); - for (int i = 0; i < length; i++) { - sink.append((char) byteValue()); - } - return sink.toString(); - } - - private int intValue() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long longValue() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private int position() { - return position; - } - - private int shortValue() { - Assert.assertTrue(position + 2 <= limit); - int value = Unsafe.getUnsafe().getShort(address + position) & 0xffff; - position += 2; - return value; - } - - private String stringValue() { - int length = varint(); - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) byteValue(); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = byteValue(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java index 74358f1f2..5ff9bba26 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java @@ -13,10 +13,8 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; @@ -25,11 +23,14 @@ import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTableHeader; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingTextTest { @@ -57,25 +58,25 @@ public void testConformanceCorpusUsesExactTargetWire() throws Exception { QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); append(binding, fields[1], value); buffer.nextRow(); - Reader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType); if (value == null) { - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.byteValue()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); if (wireType == QwpConstants.TYPE_VARCHAR) { - Assert.assertEquals(0, reader.intValue()); + Assert.assertEquals(0, reader.i32()); } else { Assert.assertEquals(0, reader.varint()); } } else if (wireType == QwpConstants.TYPE_VARCHAR) { byte[] expected = hex(fields[5]); - Assert.assertEquals(0, reader.byteValue()); - Assert.assertEquals(0, reader.intValue()); - Assert.assertEquals(expected.length, reader.intValue()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); Assert.assertArrayEquals(expected, reader.bytes(expected.length)); Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); } else { byte[] expected = hex(fields[5]); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(1, reader.varint()); Assert.assertArrayEquals(expected, reader.stringBytes()); Assert.assertEquals(0, reader.varint()); @@ -146,25 +147,25 @@ public void testFailureRollbackAndLifecycle() throws Exception { binding.symbol("a", "reset"); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); reset.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", new String(reset.stringBytes(), StandardCharsets.UTF_8)); - Assert.assertEquals(1, reset.byteValue()); - Assert.assertEquals(1, reset.intValue()); - Assert.assertEquals(1, reset.longValue()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); Assert.assertEquals(1, reset.varint()); Assert.assertEquals(2, reset.varint()); Assert.assertEquals("a", new String(reset.stringBytes(), StandardCharsets.UTF_8)); - Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.byteValue()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); Assert.assertEquals("c", new String(reset.stringBytes(), StandardCharsets.UTF_8)); - Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.byteValue()); - Assert.assertEquals(0, reset.byteValue()); - Assert.assertEquals(0, reset.intValue()); - Assert.assertEquals(5, reset.intValue()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(5, reset.i32()); Assert.assertArrayEquals("reset".getBytes(StandardCharsets.UTF_8), reset.bytes(5)); - Assert.assertEquals(1, reset.byteValue()); - Assert.assertEquals(1, reset.byteValue()); - Assert.assertEquals(0, reset.intValue()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); Assert.assertEquals(resetSize, reset.position()); buffer.clear(); assertIllegalState(() -> binding.symbol("a", "stale")); @@ -259,7 +260,7 @@ public void testOwnerBackedGlobalSymbolIdsAndEncoding() throws Exception { encoder.beginSchemaMessage(1, mirror, -1, 3); encoder.addSchemaTable(buffer, binding.getTableId(), binding.getMetadataVersion()); int size = encoder.finishMessage(); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals(0, reader.varint()); Assert.assertEquals(4, reader.varint()); @@ -268,7 +269,7 @@ public void testOwnerBackedGlobalSymbolIdsAndEncoding() throws Exception { Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 4); - Assert.assertEquals(0, reader.byteValue()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(1, reader.varint()); Assert.assertEquals(2, reader.varint()); Assert.assertEquals(3, reader.varint()); @@ -290,68 +291,13 @@ private static void append(QwpSchemaBinding binding, String inputType, CharSeque } } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(-1, columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] column(String name, int type, byte[] params) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type) - .putShort((short) params.length).put(params).array(); - } - - private static byte[] frame(byte[] payload) { - return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload).array(); - } - - private static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { - int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - byte[] frame = frame(payload.array()); - long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.length; i++) { - Unsafe.getUnsafe().putByte(address + i, frame[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.length); - } finally { - Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); skipTableHeader(reader, wireType, 1); return reader; } - private static void skipTableHeader(Reader reader, byte wireType, int rows) { - Assert.assertEquals("t", new String(reader.stringBytes(), StandardCharsets.UTF_8)); - Assert.assertEquals(1, reader.byteValue()); - Assert.assertEquals(1, reader.intValue()); - Assert.assertEquals(1, reader.longValue()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(1, reader.varint()); - Assert.assertEquals("value", new String(reader.stringBytes(), StandardCharsets.UTF_8)); - Assert.assertEquals(wireType, reader.byteValue()); - } - private static int targetType(String value) { switch (value) { case "STRING": @@ -405,73 +351,4 @@ private static String utf16(String value) { private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(reason, error.getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private int byteValue() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private byte[] bytes(int length) { - Assert.assertTrue(position + length <= limit); - byte[] value = new byte[length]; - for (int i = 0; i < length; i++) { - value[i] = (byte) byteValue(); - } - return value; - } - - private int intValue() { - Assert.assertTrue(position + Integer.BYTES <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += Integer.BYTES; - return value; - } - - private long longValue() { - Assert.assertTrue(position + Long.BYTES <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += Long.BYTES; - return value; - } - - private int position() { - return position; - } - - private void skip(int length) { - Assert.assertTrue(position + length <= limit); - position += length; - } - - private byte[] stringBytes() { - return bytes(varint()); - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = byteValue(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java index 34f72585d..88e2cdf25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java @@ -10,23 +10,23 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.temporal.ChronoUnit; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; + public class QwpSchemaBindingTimestampTextTest { private static final String CORPUS = "/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv"; @@ -66,7 +66,7 @@ public void testCorpusUsesExactVarcharBytes() throws Exception { } append.run(); buffer.nextRow(); - Reader reader = tableReader( + QwpTestWireReader reader = tableReader( encoder, encoder.encodeSchema(buffer), 1, QwpConstants.TYPE_VARCHAR); String expectedText = "".equals(fields[6]) ? "" : fields[6]; byte[] expected = expectedText.getBytes(StandardCharsets.UTF_8); @@ -74,7 +74,7 @@ public void testCorpusUsesExactVarcharBytes() throws Exception { Assert.assertEquals(0, reader.i32()); Assert.assertEquals(expected.length, reader.i32()); Assert.assertArrayEquals(expected, reader.bytes(expected.length)); - Assert.assertEquals(reader.limit, reader.position); + Assert.assertEquals(reader.limit(), reader.position()); } catch (AssertionError e) { throw new AssertionError( "case_id=" + fields[0] + ", target=" + target + ": " + e.getMessage(), e); @@ -121,27 +121,27 @@ public void testUnitsInstantNullDuplicateOmissionRollbackAndReset() { binding.timestampColumn("c", 1000, ChronoUnit.NANOS); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableHeader(encoder, size, 5, 2); + QwpTestWireReader reader = tableHeader(encoder, size, 5, 2); Assert.assertEquals("a", reader.string()); Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); Assert.assertEquals("c", reader.string()); Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); assertVarchar(reader, 0x14, "", "1970-01-01T00:00:00.000Z", "1969-12-31T23:59:59.999Z"); assertVarchar(reader, 0x0f, "1970-01-01T00:00:00.000Z"); - Assert.assertEquals(size, reader.position); + Assert.assertEquals(size, reader.position()); buffer.reset(); binding.timestampColumn("a", 0, ChronoUnit.SECONDS); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = tableHeader(encoder, resetSize, 1, 2); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); Assert.assertEquals("a", reset.string()); Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); Assert.assertEquals("c", reset.string()); Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); assertVarchar(reset, -1, "1970-01-01T00:00:00.000Z"); assertVarchar(reset, 0x01); - Assert.assertEquals(resetSize, reset.position); + Assert.assertEquals(resetSize, reset.position()); } } } @@ -183,7 +183,7 @@ public void testInferenceDesignatedAndRangeErrors() { } } - private static void assertVarchar(Reader reader, int bitmap, String... values) { + private static void assertVarchar(QwpTestWireReader reader, int bitmap, String... values) { if (bitmap < 0) { Assert.assertEquals(0, reader.u8()); } else { @@ -201,137 +201,10 @@ private static void assertVarchar(Reader reader, int bitmap, String... values) { Assert.assertArrayEquals(joined.toString().getBytes(StandardCharsets.UTF_8), reader.bytes(total)); } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, int designated, byte[]... columns) { - return new QwpSchemaBinding(buffer, known(designated, columns)); - } - - private static byte[] column(String name, int type) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) 0).array(); - } - - private static QwpSchemaResponse known(int designated, byte[]... columns) { - int length = 26; - for (byte[] column : columns) { - length += column.length; - } - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designated).putShort((short) columns.length); - for (byte[] column : columns) { - payload.put(column); - } - return decode(payload.array()); - } - - private static QwpSchemaResponse missing() { - return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING).array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) { - Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - } - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte type) { - Reader reader = tableHeader(encoder, size, rows, 1); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); Assert.assertEquals("value", reader.string()); Assert.assertEquals(type, reader.u8()); return reader; } - - private static Reader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); - Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); - Assert.assertEquals(QwpConstants.VERSION, reader.u8()); - Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); - Assert.assertEquals(1, reader.u16()); - Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); - Assert.assertEquals("t", reader.string()); - Assert.assertEquals(1, reader.u8()); - Assert.assertEquals(1, reader.i32()); - Assert.assertEquals(1, reader.i64()); - Assert.assertEquals(rows, reader.varint()); - Assert.assertEquals(columns, reader.varint()); - return reader; - } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - Assert.assertEquals(reason, Assert.assertThrows(LineSenderSchemaException.class, action::run).getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private byte[] bytes(int length) { - byte[] bytes = new byte[length]; - for (int i = 0; i < length; i++) { - bytes[i] = (byte) u8(); - } - return bytes; - } - - private int i32() { - Assert.assertTrue(position + 4 <= limit); - int value = Unsafe.getUnsafe().getInt(address + position); - position += 4; - return value; - } - - private long i64() { - Assert.assertTrue(position + 8 <= limit); - long value = Unsafe.getUnsafe().getLong(address + position); - position += 8; - return value; - } - - private void skip(int length) { - position += length; - Assert.assertTrue(position <= limit); - } - - private String string() { - return new String(bytes(varint()), StandardCharsets.UTF_8); - } - - private int u8() { - Assert.assertTrue(position < limit); - return Unsafe.getUnsafe().getByte(address + position++) & 0xff; - } - - private int u16() { - return u8() | (u8() << 8); - } - - private int varint() { - int result = 0; - int shift = 0; - int value; - do { - value = u8(); - result |= (value & 0x7f) << shift; - shift += 7; - } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java index 305f136be..680b590c5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java @@ -10,21 +10,22 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; -import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; public class QwpSchemaBindingUuidTextTest { @@ -51,7 +52,7 @@ public void testCorpusUsesCanonicalVarcharWire() throws Exception { binding.uuidColumn("value", Long.parseLong(fields[1]), Long.parseLong(fields[2])); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); if ("".equals(fields[5])) { Assert.assertEquals("", fields[6]); Assert.assertEquals(1, reader.u8()); @@ -91,7 +92,7 @@ public void testDuplicateNullOmissionRollbackAndReset() throws Exception { buffer.nextRow(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = tableReader(encoder, size, 3); + QwpTestWireReader reader = tableReader(encoder, size, 3); Assert.assertEquals(1, reader.u8()); Assert.assertEquals(5, reader.u8()); Assert.assertEquals(0, reader.i32()); @@ -115,7 +116,7 @@ public void testDuplicateNullOmissionRollbackAndReset() throws Exception { buffer.nextRow(); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.u8()); @@ -135,7 +136,7 @@ public void testDuplicateNullOmissionRollbackAndReset() throws Exception { binding.uuidColumn("a", 5, 0); buffer.nextRow(); int resetSize = encoder.encodeSchema(buffer); - Reader reset = new Reader(encoder.getBuffer().getBufferPtr(), resetSize); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); reset.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reset.string()); Assert.assertEquals(1, reset.u8()); @@ -161,7 +162,7 @@ public void testDuplicateNullOmissionRollbackAndReset() throws Exception { @Test public void testNativeInferenceAndNativeBothMinRemainUuidBytes() { - assertNativeBothMin(response(column("value", ColumnType.UUID)), true); + assertNativeBothMin(known(column("value", ColumnType.UUID)), true); assertNativeBothMin(missing(), false); } @@ -172,7 +173,7 @@ private static void assertNativeBothMin(QwpSchemaResponse schema, boolean known) binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); buffer.nextRow(); int size = encoder.encodeSchema(buffer); - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(known ? 1 : 0, reader.u8()); @@ -203,12 +204,12 @@ public void testSymbolParameterizedAndDesignatedTargetsRemainUnsupported() { () -> ordinary.uuidColumn("text", 1, 0)); } try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding designated = new QwpSchemaBinding(buffer, response(0, column("ts", ColumnType.TIMESTAMP))); + QwpSchemaBinding designated = new QwpSchemaBinding(buffer, known(0, column("ts", ColumnType.TIMESTAMP))); assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> designated.uuidColumn("ts", 1, 0)); } } - private static void assertVarcharValues(Reader reader, int bitmap, String value) { + private static void assertVarcharValues(QwpTestWireReader reader, int bitmap, String value) { Assert.assertEquals(1, reader.u8()); Assert.assertEquals(bitmap, reader.u8()); Assert.assertEquals(0, reader.i32()); @@ -216,64 +217,12 @@ private static void assertVarcharValues(Reader reader, int bitmap, String value) Assert.assertEquals(value, reader.ascii(36)); } - private static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { - return new QwpSchemaBinding(buffer, response(columns)); - } - - private static byte[] column(String name, int type) { - return column(name, type, new byte[0]); - } - - private static byte[] parameterizedColumn(String name, int type) { - return column(name, type, new byte[]{1}); - } - - private static byte[] column(String name, int type, byte[] parameters) { - byte[] bytes = name.getBytes(StandardCharsets.UTF_8); - return ByteBuffer.allocate(2 + bytes.length + 6 + parameters.length).order(ByteOrder.LITTLE_ENDIAN) - .putShort((short) bytes.length).put(bytes).putInt(type).putShort((short) parameters.length) - .put(parameters).array(); - } - - private static QwpSchemaResponse missing() { - ByteBuffer payload = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_MISSING); - return decode(payload.array()); - } - - private static QwpSchemaResponse response(byte[]... columns) { - return response(-1, columns); - } - - private static QwpSchemaResponse response(int designatedIndex, byte[]... columns) { - int length = 26; - for (byte[] column : columns) length += column.length; - ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) - .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); - for (byte[] column : columns) payload.put(column); - return decode(payload.array()); - } - - private static QwpSchemaResponse decode(byte[] payload) { - ByteBuffer frame = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) - .putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - long address = Unsafe.malloc(frame.capacity(), MemoryTag.NATIVE_DEFAULT); - try { - for (int i = 0; i < frame.capacity(); i++) Unsafe.getUnsafe().putByte(address + i, frame.array()[i]); - return QwpSchemaProtocol.decodeResponse(address, frame.capacity()); - } finally { - Unsafe.free(address, frame.capacity(), MemoryTag.NATIVE_DEFAULT); - } - } - - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { return tableReader(encoder, size, rows, "value"); } - private static Reader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { - Reader reader = new Reader(encoder.getBuffer().getBufferPtr(), size); + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); reader.skip(QwpConstants.HEADER_SIZE); Assert.assertEquals("t", reader.string()); Assert.assertEquals(1, reader.u8()); @@ -297,40 +246,4 @@ private static byte[] hex(String value) { for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); return bytes; } - - private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { - LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); - Assert.assertEquals(reason, error.getReason()); - } - - private static final class Reader { - private final long address; - private final int limit; - private int position; - - private Reader(long address, int limit) { - this.address = address; - this.limit = limit; - } - - private String ascii(int length) { return new String(bytes(length), StandardCharsets.UTF_8); } - private byte[] bytes(int length) { - byte[] value = new byte[length]; - for (int i = 0; i < length; i++) value[i] = (byte) u8(); - return value; - } - private int i32() { int value = Unsafe.getUnsafe().getInt(address + position); position += 4; return value; } - private long i64() { long value = Unsafe.getUnsafe().getLong(address + position); position += 8; return value; } - private int position() { return position; } - private void skip(int length) { position += length; Assert.assertTrue(position <= limit); } - private String string() { return ascii(varint()); } - private int u8() { Assert.assertTrue(position < limit); return Unsafe.getUnsafe().getByte(address + position++) & 0xff; } - private int varint() { - int result = 0; - int shift = 0; - int value; - do { value = u8(); result |= (value & 0x7f) << shift; shift += 7; } while ((value & 0x80) != 0); - return result; - } - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java new file mode 100644 index 000000000..b4c3e6de8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java @@ -0,0 +1,190 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +/** + * Shared mechanics for the {@code QwpSchemaBinding*Test} family: schema + * response construction and encoded-table header checks. The fixtures write + * the schema envelope by hand, independently of any production encoder, and + * leave every conversion expectation to the individual tests. + * + *

      All fixtures use table {@code "t"}, table id 1 and metadata version 1 + * unless a parameter says otherwise.

      + */ +final class QwpSchemaTestFixtures { + private static final byte[] NO_PARAMS = new byte[0]; + + private QwpSchemaTestFixtures() { + } + + /** Asserts a non-retryable schema rejection whose message contains every part. */ + static void assertReason(LineSenderSchemaException.Reason reason, Runnable action, String... messageParts) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + for (String part : messageParts) { + Assert.assertTrue(error.getMessage(), error.getMessage().contains(part)); + } + } + + static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return binding(buffer, -1, columns); + } + + static QwpSchemaBinding binding(QwpTableBuffer buffer, int designatedIndex, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designatedIndex, columns)); + } + + static byte[] column(String name, int type) { + return column(name, type, NO_PARAMS); + } + + static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type) + .putShort((short) params.length).put(params).array(); + } + + /** Wraps a control payload in a message frame and decodes it. */ + static QwpSchemaResponse decode(byte[] payload) { + byte[] frame = frame(payload); + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + static byte[] frame(byte[] payload) { + return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put(QwpConstants.VERSION).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload).array(); + } + + static QwpSchemaResponse known(byte[]... columns) { + return known(-1, columns); + } + + static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + static QwpSchemaResponse missing() { + return schemaResult(QwpSchemaProtocol.RESULT_MISSING); + } + + /** A column whose type carries a non-empty extension parameter block. */ + static byte[] parameterizedColumn(String name, int type) { + return column(name, type, new byte[]{1}); + } + + /** Decodes a schema-less response carrying only {@code result}. */ + static QwpSchemaResponse schemaResult(int result) { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) result).array()); + } + + /** Consumes the table prefix and the header of a single column named "value". */ + static void skipTableHeader(QwpTestWireReader reader, byte wireType, int rows) { + skipTablePrefix(reader, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + } + + static void skipTablePrefix(QwpTestWireReader reader, int rows, int columns) { + skipTablePrefix(reader, rows, columns, 1, 1); + } + + /** Asserts the message header and table prefix, leaving the reader at the first column. */ + static QwpTestWireReader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + return tableHeader(encoder, size, rows, columns, 1, 1); + } + + /** A negative {@code tableId} expects a schema-framed table block without a pinned identity. */ + static QwpTestWireReader tableHeader( + QwpWebSocketEncoder encoder, + int size, + int rows, + int columns, + int tableId, + long metadataVersion + ) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + skipTablePrefix(reader, rows, columns, tableId, metadataVersion); + return reader; + } + + private static void skipTablePrefix( + QwpTestWireReader reader, + int rows, + int columns, + int tableId, + long metadataVersion + ) { + Assert.assertEquals("t", reader.string()); + if (tableId < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(tableId, reader.i32()); + Assert.assertEquals(metadataVersion, reader.i64()); + } + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java new file mode 100644 index 000000000..94f2da358 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java @@ -0,0 +1,140 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.std.Unsafe; +import org.junit.Assert; + +import java.nio.charset.StandardCharsets; + +/** + * Bounds-checked cursor over native memory holding an encoded QWP message. + * Tests decode the wire bytes with this reader rather than with production + * decoders, so an encoder bug cannot hide behind a matching decoder bug. + * Multi-byte reads use the native byte order, like the production encoder. + */ +final class QwpTestWireReader { + private final long address; + private final int limit; + private int position; + + QwpTestWireReader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + /** Reads {@code length} raw bytes as a string, without a length prefix. */ + String ascii(int length) { + return new String(bytes(length), StandardCharsets.UTF_8); + } + + byte[] bytes(int length) { + require(length); + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = Unsafe.getUnsafe().getByte(address + position + i); + } + position += length; + return value; + } + + float f32() { + return Float.intBitsToFloat(i32()); + } + + double f64() { + return Double.longBitsToDouble(i64()); + } + + short i16() { + require(Short.BYTES); + short value = Unsafe.getUnsafe().getShort(address + position); + position += Short.BYTES; + return value; + } + + int i32() { + require(Integer.BYTES); + int value = Unsafe.getUnsafe().getInt(address + position); + position += Integer.BYTES; + return value; + } + + long i64() { + require(Long.BYTES); + long value = Unsafe.getUnsafe().getLong(address + position); + position += Long.BYTES; + return value; + } + + int limit() { + return limit; + } + + int position() { + return position; + } + + void skip(int length) { + require(length); + position += length; + } + + /** Reads a varint-prefixed UTF-8 string. */ + String string() { + return new String(stringBytes(), StandardCharsets.UTF_8); + } + + /** Reads the raw bytes of a varint-prefixed string. */ + byte[] stringBytes() { + return bytes(varint()); + } + + int u16() { + return i16() & 0xffff; + } + + int u8() { + require(1); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + int varint() { + int result = 0; + for (int shift = 0; shift < 32; shift += 7) { + int value = u8(); + result |= (value & 0x7f) << shift; + if ((value & 0x80) == 0) { + return result; + } + } + throw new AssertionError("varint exceeds 32 bits at position " + position); + } + + private void require(int length) { + Assert.assertTrue( + "read of " + length + " bytes at " + position + " exceeds limit " + limit, + length >= 0 && position <= limit - length + ); + } +} From 0fb1ea924b93880da3d421bc1ac43f5157912c89 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 18 Sep 2026 10:51:46 +0200 Subject: [PATCH 26/51] Pin one schema snapshot per table per batch The schema-aware sender no longer swaps a table's schema binding under pending rows. Once a table holds rows in the current batch, every later row of that batch validates against the snapshot the first row pinned; bindingForEffectiveWrite() consults the cache only when the table buffer is row-less, which is the first row after a flush or reset. installSchemaBinding() then clears the buffer and rebinds it in place instead of retiring the old buffer alongside a new one. A failed setter reports its local INVALID_VALUE or UNSUPPORTED_FEATURE directly. onRowFailure() rolls the row back and rethrows; it never performs a network lookup, so a rejection costs no round trip and cannot fail for a transport reason. The cache stays current through the paths that already exist: the server attaches the table's current schema to the ACK of a frame that carries a stale identity, the send loop applies that feedback before it advances the ack watermark, INVALIDATE_ALL empties the cache so the next batch looks the table up again, and a reconnect clears the cache. An explicit refresh API was considered and dropped: a caller cannot tell a stale-snapshot rejection from invalid input, so there is no sound rule for when to call it. This removes retiredTableBuffers and its handling on flush collection, post-flush reset, reset(), close() and byte accounting; the schemaResolutionFresh out-parameter threaded through resolveSchema() and QwpSchemaCoordinator.resolve(); refreshAfterSchemaRejection(); QwpSchemaResponse .hasSameRelevantTarget() and its binding delegate; and the SCHEMA_CHANGED reason, which nothing raises any more. The integration test fixtures now acknowledge like the real server: SchemaHandler and LongTextSchemaHandler attach the current schema to the ACK when the snapshot they last delivered is stale, or answer INVALIDATE_ALL when they cannot describe. The generation rebind tests drive adoption through flushAndGetSequence() and awaitAckedFsn() and read the two generations from two single-block frames via a multi-frame FrameReader. New coverage: an INVALIDATE_ALL ACK makes the next batch surface the typed lookup failure from the setter, a version change does not split the pending batch, and reset() keeps the cached snapshot because it ships no frame. The design doc's "Local errors and stale schemas" and "One schema snapshot per table per batch" sections describe the new contract, and the contract-simplification proposal records proposal 1 as implemented without an explicit refresh. Co-Authored-By: Claude Fable 5.1 --- .../client/LineSenderSchemaException.java | 1 - .../qwp/client/QwpWebSocketSender.java | 220 ++---- .../sf/cursor/CursorWebSocketSendLoop.java | 7 - .../sf/cursor/QwpSchemaCoordinator.java | 16 - .../qwp/protocol/QwpSchemaBinding.java | 4 - .../qwp/protocol/QwpSchemaResponse.java | 17 - .../QwpSchemaSenderIntegrationTest.java | 737 ++++++++++++------ .../qwp/protocol/QwpSchemaProtocolTest.java | 48 -- ...ma-aware-sender-contract-simplification.md | 221 ++++++ design/schema-aware-sender.md | 136 ++-- 10 files changed, 854 insertions(+), 553 deletions(-) create mode 100644 design/schema-aware-sender-contract-simplification.md diff --git a/core/src/main/java/io/questdb/client/LineSenderSchemaException.java b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java index 4fd6e5121..3fe7fa5de 100644 --- a/core/src/main/java/io/questdb/client/LineSenderSchemaException.java +++ b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java @@ -29,7 +29,6 @@ public class LineSenderSchemaException extends LineSenderException { public enum Reason { INVALID_VALUE, - SCHEMA_CHANGED, SCHEMA_UNAVAILABLE, ACCESS_DENIED, UNSUPPORTED_FEATURE diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 6c5820caf..4679bef4d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -50,7 +50,6 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; -import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; @@ -205,8 +204,6 @@ public class QwpWebSocketSender implements Sender { // lockstep so index i names the same table in both, and in the same order as // splitFrameBodyBytes, which is what lets the split passes index them directly. private final ObjList flushTableBuffers = new ObjList<>(); - private final ObjList retiredTableBuffers = new ObjList<>(); - private final boolean[] schemaResolutionFresh = new boolean[1]; private final ObjList flushTableNames = new ObjList<>(); private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; @@ -1044,7 +1041,7 @@ public void at(long timestamp, ChronoUnit unit) { atMicros(micros); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, null, true); + throw onRowFailure(e); } } @@ -1062,7 +1059,7 @@ public void at(Instant timestamp) { long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; atMicros(micros); } catch (RuntimeException | Error e) { - throw onRowFailure(e, null, true); + throw onRowFailure(e); } } @@ -1075,7 +1072,7 @@ public void atNow() { // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { - throw onRowFailure(e, null); + throw onRowFailure(e); } } @@ -1156,7 +1153,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { col.addBinary(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1183,7 +1180,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice } return binaryColumn(columnName, slice.ptr(), slice.size()); } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } } @@ -1207,7 +1204,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l col.addBinary(ptr, len); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1227,7 +1224,7 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { col.addBoolean(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1258,7 +1255,7 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { col.addByte(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1295,7 +1292,7 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { col.addShort((short) value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1525,10 +1522,6 @@ private void close0(boolean[] restoreInterrupt) { } } tableBuffers.clear(); - for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { - Misc.free(retiredTableBuffers.getQuick(i)); - } - retiredTableBuffers.clear(); } catch (Throwable t) { LOG.error("Error closing encoder or table buffers: {}", String.valueOf(t)); terminalError = captureCloseError(terminalError, t); @@ -1639,7 +1632,7 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { col.addDecimal64(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1660,7 +1653,7 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { col.addDecimal128(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1681,7 +1674,7 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { col.addDecimal256(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1703,7 +1696,7 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { col.addDecimal256(currentDecimal256); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1724,7 +1717,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1745,7 +1738,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1766,7 +1759,7 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1787,7 +1780,7 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { col.addDoubleArray(array); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, name); + throw onRowFailure(e); } return this; } @@ -1807,7 +1800,7 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { col.addDouble(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1833,7 +1826,7 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { col.addFloat(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -1983,7 +1976,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int col.addGeoHash(maskGeoHashBits(bits, precisionBits), precisionBits); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -2013,7 +2006,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va return this; } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } if (value == null) { throw new LineSenderException( @@ -2405,7 +2398,7 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { col.addInt(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -2438,7 +2431,7 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { col.addIPv4(address); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -2484,7 +2477,7 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre return this; } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { throw new LineSenderException( @@ -2532,7 +2525,7 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l col.addLong256(l0, l1, l2, l3); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -2554,7 +2547,7 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { } } catch (RuntimeException | Error e) { // LONG_ARRAY is unimplemented; a schema refresh cannot make it valid. - throw onRowFailure(e, null); + throw onRowFailure(e); } return this; } @@ -2575,7 +2568,7 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, null); + throw onRowFailure(e); } return this; } @@ -2596,7 +2589,7 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, null); + throw onRowFailure(e); } return this; } @@ -2617,7 +2610,7 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { col.addLongArray(array); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, null); + throw onRowFailure(e); } return this; } @@ -2637,7 +2630,7 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { col.addLong(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -2718,10 +2711,6 @@ public void reset() { buf.reset(); } } - for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { - Misc.free(retiredTableBuffers.getQuick(i)); - } - retiredTableBuffers.clear(); // Drop the batch's symbol watermark along with the rows that raised it. A // later flush encodes the delta section as // [sentMaxSymbolId+1 .. currentBatchMaxSymbolId], so a watermark left behind @@ -2954,7 +2943,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { col.addShort(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -3074,7 +3063,7 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val col.addString(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -3094,7 +3083,7 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { col.addSymbol(value); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -3155,7 +3144,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C } } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -3176,7 +3165,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value col.addLong(micros); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -3203,7 +3192,7 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) col.addUuid(hi, lo); } } catch (RuntimeException | Error e) { - throw onRowFailure(e, columnName); + throw onRowFailure(e); } return this; } @@ -3983,13 +3972,6 @@ private synchronized Throwable closeRemainingResources(Throwable terminalError) private int collectNonEmptyTables(ObjList keys) { flushTableNames.clear(); flushTableBuffers.clear(); - for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { - QwpTableBuffer tableBuffer = retiredTableBuffers.getQuick(i); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - flushTableNames.add(tableBuffer.getTableName()); - flushTableBuffers.add(tableBuffer); - } - } for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -4382,14 +4364,21 @@ private void ensureConnected() { * fresh senders use the legacy contract until schema support is confirmed; * recovered schema-framed data and senders that have already confirmed support * retain the strict schema path. + *

      + * A table pins one schema snapshot per batch. Once the table holds pending rows, + * later rows of the same batch validate against that snapshot without consulting + * the cache; the first row after a flush or reset adopts whatever the cache holds + * by then, which ACK schema feedback and reconnects keep current. */ private QwpSchemaBinding bindingForEffectiveWrite() { QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); if (currentTableBuffer.hasInProgressRow()) { - schemaResolutionFresh[0] = false; return pinned; } ensureConnected(); + if (pinned != null && currentTableBuffer.getRowCount() > 0) { + return pinned; + } // A batch has one wire contract, because a QWP frame has one wire family. // While the pending batch holds legacy rows, later rows stay legacy even if // a supporting handshake has landed since; the first row after the flush @@ -4398,7 +4387,6 @@ private QwpSchemaBinding bindingForEffectiveWrite() { if ((pendingRowCount > 0 && isPendingBatchLegacy) || (initialConnectMode == Sender.InitialConnectMode.ASYNC && !cursorEngine.requiresSchema())) { - schemaResolutionFresh[0] = false; return null; } final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; @@ -4406,113 +4394,43 @@ private QwpSchemaBinding bindingForEffectiveWrite() { return null; } QwpSchemaResponse latest = cursorSendLoop.resolveSchema( - currentTableName, remainingSchemaMillis(deadlineNanos), schemaResolutionFresh); + currentTableName, remainingSchemaMillis(deadlineNanos)); if (pinned != null && pinned.getTableId() == latest.getTableId() && pinned.getMetadataVersion() == latest.getMetadataVersion()) { return pinned; } - return installSchemaBinding(latest, pinned); + return installSchemaBinding(latest); } - private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest, QwpSchemaBinding pinned) { - // Bind in place only a pristine buffer. A flushed legacy buffer has no rows - // but keeps its legacy column layout, which a schema binding cannot adopt; - // the replacement below covers it. - if (pinned == null && currentTableBuffer.getRowCount() == 0 - && currentTableBuffer.getColumnCount() == 0) { - return new QwpSchemaBinding(currentTableBuffer, latest); - } - QwpTableBuffer replacement = new QwpTableBuffer(currentTableName, this); - final QwpSchemaBinding replacementBinding; - try { - replacementBinding = new QwpSchemaBinding(replacement, latest); - } catch (RuntimeException | Error e) { - replacement.close(); - throw e; - } - QwpTableBuffer old = currentTableBuffer; - int retiredSlot = -1; - try { - if (old.getRowCount() > 0) { - retiredSlot = retiredTableBuffers.size(); - // Reserve list capacity before replacing the map entry. After the - // map mutation, the remaining assignments cannot allocate. - retiredTableBuffers.add(null); - } - tableBuffers.put(currentTableName, replacement); - } catch (RuntimeException | Error e) { - if (retiredSlot >= 0) { - retiredTableBuffers.remove(retiredSlot); - } - replacement.close(); - throw e; - } - currentTableBuffer = replacement; - if (retiredSlot >= 0) { - retiredTableBuffers.setQuick(retiredSlot, old); - } else { - old.close(); + /** + * Binds the row-less current table buffer to {@code latest}. A buffer that has + * been bound before, or that a flushed legacy batch left with a column layout + * ({@link QwpTableBuffer#reset()} keeps columns), cannot take a new binding in + * place; {@link QwpTableBuffer#clear()} drops both before rebinding. + */ + private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest) { + assert currentTableBuffer.getRowCount() == 0 : "schema binding replaced under pending rows"; + if (currentTableBuffer.getSchemaBinding() != null || currentTableBuffer.getColumnCount() != 0) { + currentTableBuffer.clear(); + currentTableBufferSnapshotBytes = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; } - currentTableBufferSnapshotBytes = 0; - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - return replacementBinding; - } - - // Single home of the row-failure policy: roll the whole row back first, so a - // failed refresh cannot leave a half-written row, then apply the one-refresh - // rule to a schema rejection. Every other failure propagates unchanged. - private RuntimeException onRowFailure(Throwable e, CharSequence columnName) { - return onRowFailure(e, columnName, false); + return new QwpSchemaBinding(currentTableBuffer, latest); } - private RuntimeException onRowFailure(Throwable e, CharSequence columnName, boolean isDesignatedTimestamp) { + // Single home of the row-failure policy: roll the whole row back, then report + // the failure as raised. A schema rejection is final for the batch's snapshot; + // the sender never looks up fresh metadata from a failed setter. + private RuntimeException onRowFailure(Throwable e) { rollbackRow(); - if (e instanceof LineSenderSchemaException) { - return refreshAfterSchemaRejection((LineSenderSchemaException) e, columnName, isDesignatedTimestamp); - } if (e instanceof Error) { throw (Error) e; } return (RuntimeException) e; } - private RuntimeException refreshAfterSchemaRejection( - LineSenderSchemaException rejection, - CharSequence columnName, - boolean designatedTimestamp - ) { - if (schemaResolutionFresh[0] - || (rejection.getReason() != LineSenderSchemaException.Reason.INVALID_VALUE - && rejection.getReason() != LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) - || (!designatedTimestamp && (columnName == null - || !TableUtils.isValidColumnName(columnName, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)))) { - return rejection; - } - QwpSchemaBinding old = currentTableBuffer.getSchemaBinding(); - // An unbound failure has no pinned target to compare. Preserve its typed - // error instead of replacing it with a refresh failure or null dereference. - if (old == null) { - return rejection; - } - QwpSchemaResponse fresh = cursorSendLoop.refreshSchema(currentTableName, schemaWaitMillis); - boolean identityChanged = old.getTableId() != fresh.getTableId() - || old.getMetadataVersion() != fresh.getMetadataVersion(); - boolean changed = old.getTableId() != fresh.getTableId() - || !old.hasSameRelevantTarget(fresh, columnName); - if (identityChanged) { - installSchemaBinding(fresh, old); - } - if (!changed) { - return rejection; - } - return new LineSenderSchemaException( - LineSenderSchemaException.Reason.SCHEMA_CHANGED, - "schema changed while rejecting row [table=" + currentTableName - + ", column=" + columnName + "]"); - } - private static long remainingSchemaMillis(long deadlineNanos) { long remaining = deadlineNanos - System.nanoTime(); return remaining <= 0 ? 0 : 1L + (remaining - 1L) / 1_000_000L; @@ -4917,10 +4835,6 @@ private void resetTableBuffersAfterFlush() { // Drop the references; the next flush re-collects. flushTableNames.clear(); flushTableBuffers.clear(); - for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { - Misc.free(retiredTableBuffers.getQuick(i)); - } - retiredTableBuffers.clear(); currentBatchMaxSymbolId = -1; pendingBytes = 0; currentTableBufferSnapshotBytes = 0; @@ -5676,12 +5590,6 @@ private long toMicros(long value, ChronoUnit unit) { @TestOnly public long totalBufferedBytes() { long total = 0; - for (int i = 0, n = retiredTableBuffers.size(); i < n; i++) { - QwpTableBuffer retired = retiredTableBuffers.getQuick(i); - if (retired != null) { - total += retired.getBufferedBytes(); - } - } ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index b47870299..811ad2431 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1227,13 +1227,6 @@ public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMilli return schemaCoordinator.resolve(tableName, timeoutMillis, false, ioThread); } - public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMillis, boolean[] freshOut) { - if (freshOut == null || freshOut.length == 0) { - throw new IllegalArgumentException("freshOut must contain one element"); - } - return schemaCoordinator.resolve(tableName, timeoutMillis, false, ioThread, freshOut); - } - /** * Evicts the named cache entry at admission and resolves it again through the I/O thread. * This producer-side API permits one outstanding lookup. The timeout covers the entire diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index 1c54a2900..367dd59eb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -30,16 +30,6 @@ final class QwpSchemaCoordinator { private Request request; QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean refresh, Thread ioThread) { - return resolve(tableName, timeoutMillis, refresh, ioThread, null); - } - - QwpSchemaResponse resolve( - CharSequence tableName, - long timeoutMillis, - boolean refresh, - Thread ioThread, - boolean[] freshOut - ) { final long startNanos = System.nanoTime(); if (timeoutMillis < 0) { throw new IllegalArgumentException("timeoutMillis must be non-negative"); @@ -58,9 +48,6 @@ QwpSchemaResponse resolve( if (!refresh) { QwpSchemaResponse cached = cache == null ? null : cache.get(key); if (cached != null) { - if (freshOut != null) { - freshOut[0] = false; - } return cached; } } else { @@ -76,9 +63,6 @@ QwpSchemaResponse resolve( throw failure(UNSUPPORTED_FEATURE, key, "schema request id space is exhausted"); } long id = nextRequestId; - if (freshOut != null) { - freshOut[0] = true; - } byte[] message = QwpSchemaProtocol.encodeDescribe(id, tableName); nextRequestId++; Request own = new Request(id, key, startNanos, timeoutNanos, message); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 45047a52f..dedf26a6f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -903,10 +903,6 @@ public long getMetadataVersion() { return schema.getMetadataVersion(); } - public boolean hasSameRelevantTarget(QwpSchemaResponse other, CharSequence name) { - return schema.hasSameRelevantTarget(other, name); - } - private LineSenderSchemaException error( LineSenderSchemaException.Reason reason, CharSequence column, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java index 92285dc39..49fa5883d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java @@ -104,23 +104,6 @@ public int getTableId() { return tableId; } - /** - * Returns whether {@code name} resolves to an equivalent conversion target - * in both snapshots: same presence, designated-timestamp role, column type, - * and extension-parameter flag. A null {@code name} selects the designated - * timestamp. Column position is irrelevant. - */ - public boolean hasSameRelevantTarget(QwpSchemaResponse other, CharSequence name) { - int current = name == null ? designatedIndex : getColumnIndex(name); - int next = name == null ? other.designatedIndex : other.getColumnIndex(name); - if (current < 0 || next < 0) { - return current < 0 && next < 0; - } - return (current == designatedIndex) == (next == other.designatedIndex) - && columns[current].type == other.columns[next].type - && columns[current].hasTypeParameters == other.columns[next].hasTypeParameters; - } - public boolean hasSchema() { return result == QwpSchemaProtocol.RESULT_KNOWN; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index af5b9e5d2..f77291b2d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -12,6 +12,7 @@ import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.std.Decimal128; @@ -34,6 +35,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -76,7 +78,7 @@ public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exc new FrameReader(handler.awaitDataFrame()).decimal64Table( "events", 921, 931, 2, 120, 250); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } }); @@ -143,7 +145,7 @@ public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() thr new FrameReader(handler.awaitDataFrame()).smallIntegerNumericTable( "events", 901 + i, 911 + i, numericWireType(target)); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } } @@ -162,16 +164,21 @@ public void testIntegerTemporalGenerationRemainsDateBeforeTimestampRebind() thro sender.table("events"); try { sender.stringColumn("value", "1970-01-01T00:00:00.000001Z"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.intColumn("value", 12).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").intColumn("value", 12).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()) + new FrameReader(handler.awaitDataFrames(2)) .twoDateAndTimestampBlocks("events", 981, 991, 992); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -202,7 +209,7 @@ public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exc new FrameReader(handler.awaitDataFrame()).integerTemporalTable( "events", 961 + i, 971 + i, temporalWireType(target)); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } } @@ -221,16 +228,21 @@ public void testSmallIntegerTextGenerationRemainsSymbolBeforeVarcharRebind() thr sender.table("events"); try { sender.uuidColumn("value", 1, 2); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.shortColumn("value", (short) 8).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").shortColumn("value", (short) 8).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()) + new FrameReader(handler.awaitDataFrames(2)) .twoSymbolAndVarcharBlocks("events", 1021, 1031, 1032); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -260,7 +272,7 @@ public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Ex new FrameReader(handler.awaitDataFrame()).smallIntegerTextTable( "events", 1041 + i, 1051 + i, target); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } } @@ -280,16 +292,21 @@ public void testSmallIntegerDecimalGenerationRemainsPinnedBeforeRebind() throws sender.table("events"); try { sender.shortColumn("value", (short) 1_000); - Assert.fail("expected width-and-scale-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.shortColumn("value", (short) 1_000).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").shortColumn("value", (short) 1_000).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoSmallIntegerDecimalBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoSmallIntegerDecimalBlocks( "events", 1061, 1071, 1072); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -317,7 +334,7 @@ public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Excep new FrameReader(handler.awaitDataFrame()).smallIntegerDecimalTable( "events", 1081, 1091); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } }); @@ -336,16 +353,21 @@ public void testIntToIpv4GenerationRemainsPinnedBeforeRebind() throws Exception sender.table("events"); try { sender.intColumn("value", 1_000); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.intColumn("value", 1_000).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").intColumn("value", 1_000).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoIntDecimalAndIpv4Blocks( + new FrameReader(handler.awaitDataFrames(2)).twoIntDecimalAndIpv4Blocks( "events", 1101, 1111, 1112); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -371,7 +393,7 @@ public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { sender.flush(); new FrameReader(handler.awaitDataFrame()).intToIpv4Table("events", 1121, 1131); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } }); @@ -519,14 +541,19 @@ public void testNarrowDecimalGenerationRemainsPinnedBeforeWideRebind() throws Ex sender.table("events"); try { sender.decimalColumn("value", Decimal128.fromLong(1_000, 0)); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.decimalColumn("value", Decimal128.fromLong(1_000, 0)).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", Decimal128.fromLong(1_000, 0)).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDecimalBlocks("events", 621, 631, 632); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalBlocks("events", 621, 631, 632); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -543,15 +570,20 @@ public void testDecimalScaleGenerationRemainsPinnedBeforeScaleRebind() throws Ex sender.table("events"); try { sender.decimalColumn("value", new Decimal64(12_345, 4)); - Assert.fail("expected scale-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.decimalColumn("value", new Decimal64(12_345, 4)).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", new Decimal64(12_345, 4)).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDecimal64ScaleBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoDecimal64ScaleBlocks( "events", 641, 651, 652); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -568,14 +600,19 @@ public void testLongDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() thr sender.table("events"); try { sender.longColumn("value", 100_000_000_000_000L); - Assert.fail("expected width-and-scale-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.longColumn("value", 100_000_000_000_000L).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 100_000_000_000_000L).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDecimalWidthAndScaleBlocks("events", 661, 671, 672); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events", 661, 671, 672); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -592,14 +629,19 @@ public void testStringDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() t sender.table("events"); try { sender.stringColumn("value", "100000000000000"); - Assert.fail("expected width-and-scale-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.stringColumn("value", "100000000000000").atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "100000000000000").atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDecimalWidthAndScaleBlocks("events", 681, 691, 692); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events", 681, 691, 692); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -616,14 +658,19 @@ public void testFloatingDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() sender.table("events"); try { sender.doubleColumn("value", 100_000_000_000_000.0); - Assert.fail("expected width-and-scale-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.doubleColumn("value", 100_000_000_000_000.0).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleColumn("value", 100_000_000_000_000.0).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDecimalWidthAndScaleBlocks("events", 701, 711, 712); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events", 701, 711, 712); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -640,15 +687,20 @@ public void testNativeDecimalTextGenerationRemainsPinnedBeforeRebind() throws Ex sender.table("events"); try { sender.decimalColumn("value", new Decimal64(250_000, 4)); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.decimalColumn("value", new Decimal64(250_000, 4)).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", new Decimal64(250_000, 4)).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDecimal64AndVarcharBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoDecimal64AndVarcharBlocks( "events", 741, 751, 752); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -696,14 +748,19 @@ public void testGeoHashPrecisionGenerationRemainsPinnedBeforeRebind() throws Exc sender.table("events"); try { sender.stringColumn("value", "z"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.stringColumn("value", "z").atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "z").atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoGeoHashBlocks("events", 571, 581, 582); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoGeoHashBlocks("events", 571, 581, 582); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -742,14 +799,19 @@ public void testLong256GenerationRemainsPinnedBeforeVarcharRebind() throws Excep sender.table("events"); try { sender.stringColumn("value", "not-long256"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.stringColumn("value", "not-long256").atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "not-long256").atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoLong256AndVarcharBlocks("events", 451, 461, 462); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoLong256AndVarcharBlocks("events", 451, 461, 462); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -785,14 +847,19 @@ public void testVarcharGenerationRemainsPinnedBeforeCharRebind() throws Exceptio sender.table("events"); try { sender.binaryColumn("value", new byte[]{1}); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.charColumn("value", '\uffff').atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").charColumn("value", '\uffff').atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoVarcharAndCharBlocks("events", 411, 421, 422); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoVarcharAndCharBlocks("events", 411, 421, 422); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -831,15 +898,20 @@ public void testNativeTimestampGenerationRemainsPinnedBeforeVarcharRebind() thro sender.table("events"); try { sender.uuidColumn("value", 1, 0); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.NANOS).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").timestampColumn("value", Long.MAX_VALUE, ChronoUnit.NANOS).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoTimestampAndVarcharBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoTimestampAndVarcharBlocks( "events", 371, 381, 382); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -895,7 +967,8 @@ public void testStringToDateUsesExactTargetWireAndRollback() throws Exception { sender.flush(); new FrameReader(handler.awaitDataFrame()) .stringDateTable("events", 1141, 1151, -86_400_000L); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } }); } @@ -911,15 +984,20 @@ public void testDateGenerationRemainsPinnedBeforeVarcharRebind() throws Exceptio sender.table("events"); try { sender.stringColumn("value", "native-a"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.stringColumn("value", "native-a").atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "native-a").atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()) + new FrameReader(handler.awaitDataFrames(2)) .twoDateAndVarcharBlocks("events", 1161, 1171, 1172); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -944,7 +1022,7 @@ public void testDoubleArrayUsesExactTargetWireAndRollback() throws Exception { sender.flush(); new FrameReader(handler.awaitDataFrame()) .doubleArrayTable("events", 1181, 1191); - Assert.assertEquals("setter must use the standard one-refresh path", 2, + Assert.assertEquals("a rejected setter must not look up metadata", 1, handler.describeRequests.get()); } }); @@ -963,15 +1041,20 @@ public void testDoubleArrayGenerationRemainsPinnedAcrossRankChange() throws Exce sender.table("events"); try { sender.doubleArray("value", new double[]{3.0, 4.0}); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.doubleArray("value", new double[]{3.0, 4.0}).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleArray("value", new double[]{3.0, 4.0}).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()) + new FrameReader(handler.awaitDataFrames(2)) .twoDoubleArrayRankBlocks("events", 1201, 1211, 1212); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1021,7 +1104,7 @@ public void testLongArrayRejectionRollsBackWithoutRefresh() throws Exception { } @Test - public void testLongArrayRejectionDoesNotRefreshAfterSchemaChange() throws Exception { + public void testLongArrayRejectionKeepsPinnedSnapshotAfterSchemaChange() throws Exception { assertMemoryLeak(() -> { int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); @@ -1041,20 +1124,26 @@ public void testLongArrayRejectionDoesNotRefreshAfterSchemaChange() throws Excep Assert.assertEquals("schema changes cannot implement LONG_ARRAY", 1, handler.describeRequests.get()); } - // A supported input still discovers the rank change and can be retried. + // A supported input of the new rank is still rejected against the + // pinned snapshot; the batch does not discover the rank change. try { sender.doubleArray("value", new double[]{3.0, 4.0}); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.doubleArray("value", new double[]{3.0, 4.0}).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleArray("value", new double[]{3.0, 4.0}).atNow(); sender.flush(); // The rebound block holds the recovered row alone: rollback dropped - // failed_b before the refresh installed the new generation. - new FrameReader(handler.awaitDataFrame()) + // failed_b with the rejected row. + new FrameReader(handler.awaitDataFrames(2)) .twoDoubleArrayRankBlocks("events", 1241, 1251, 1252); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1090,16 +1179,21 @@ public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exc sender.table("events"); try { sender.stringColumn("failed_b", "not-a-uuid"); - Assert.fail("expected invalid UUID and schema adoption"); + Assert.fail("expected invalid UUID"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.stringColumn("value", "1970-01-01T00:00:00.000001Z").atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "1970-01-01T00:00:00.000001Z").atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoVarcharAndTimestampBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoVarcharAndTimestampBlocks( "events", 321, 331, 332 ); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1123,7 +1217,8 @@ public void testLongToTimestampTargetsUseRawUnitsAndRollback() throws Exception sender.flush(); new FrameReader(handler.awaitDataFrame()) .longTimestampTable("events", 251 + target, 261, target, Long.MAX_VALUE); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } }); @@ -1140,14 +1235,19 @@ public void testLongGenerationRemainsNativeBeforeTimestampRebind() throws Except sender.table("events"); try { sender.stringColumn("value", "not-a-long"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.longColumn("value", 12).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 12).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoLongAndTimestampBlocks("events", 271, 281, 282); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoLongAndTimestampBlocks("events", 271, 281, 282); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1163,14 +1263,19 @@ public void testDoubleGenerationRemainsNativeBeforeVarcharRebind() throws Except sender.table("events"); try { sender.stringColumn("value", "not-a-double"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.doubleColumn("value", 1e23).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleColumn("value", 1e23).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoDoubleAndVarcharBlocks("events", 231, 241, 242); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoDoubleAndVarcharBlocks("events", 231, 241, 242); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1207,7 +1312,8 @@ public void testFloatingToTextTargetsUsePinnedWireAndRollback() throws Exception "FLOAT".equals(input) ? "0.10000000149011612" : "1.0E23", "FLOAT".equals(input) ? "1.401298464324817E-45" : "5.0E-324" ); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } } @@ -1229,15 +1335,20 @@ public void testUuidGenerationRemainsNativeBeforeVarcharRebind() throws Exceptio sender.table("events"); try { sender.stringColumn("value", "not-a-uuid"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.uuidColumn("value", 0x2122232425262728L, 0x3132333435363738L).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("value", 0x2122232425262728L, 0x3132333435363738L).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoUuidAndVarcharBlocks("events", 191, 201, 202); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoUuidAndVarcharBlocks("events", 191, 201, 202); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1272,7 +1383,8 @@ public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti target, "11121314-1516-1718-0102-030405060708" ); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } }); @@ -1312,7 +1424,8 @@ public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { new FrameReader(handler.awaitDataFrame()).ipv4Table( "events", 211 + target, 221, target, 0xff010203); - Assert.assertEquals(3, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } }); @@ -1331,15 +1444,20 @@ public void testIpv4GenerationRemainsNativeBeforeVarcharRebind() throws Exceptio sender.table("events"); try { sender.ipv4Column("value", "not-an-ip"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.ipv4Column("value", 0x05060708).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").ipv4Column("value", 0x05060708).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoIpv4AndVarcharBlocks("events", 231, 241, 242); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2)).twoIpv4AndVarcharBlocks("events", 231, 241, 242); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1370,7 +1488,8 @@ public void testGeoHashTargetsUsePinnedTargetWireAndRollback() throws Exception new FrameReader(handler.awaitDataFrame()).geoHashTargetTable( "events", 251 + target, 261, target, 0xd0c6c, 0x12345, 20); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } }); @@ -1390,16 +1509,21 @@ public void testGeoHashGenerationRemainsNativeBeforeVarcharRebind() throws Excep sender.table("events"); try { sender.stringColumn("value", "not-geohash"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.geoHashColumn("value", 0x12345, 20).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").geoHashColumn("value", 0x12345, 20).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoNativeGeoHashAndVarcharBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoNativeGeoHashAndVarcharBlocks( "events", 281, 291, 292); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1430,7 +1554,8 @@ public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception new FrameReader(handler.awaitDataFrame()).long256TargetTable( "events", 251 + target, 261, target, 1, 2, 3, 4); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } }); @@ -1449,16 +1574,21 @@ public void testLong256GenerationRemainsNativeBeforeVarcharRebind() throws Excep sender.table("events"); try { sender.stringColumn("value", "not-long256"); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.long256Column("value", 5, 6, 7, 8).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").long256Column("value", 5, 6, 7, 8).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()).twoNativeLong256AndVarcharBlocks( + new FrameReader(handler.awaitDataFrames(2)).twoNativeLong256AndVarcharBlocks( "events", 281, 291, 292); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1489,7 +1619,8 @@ public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti FrameReader reader = new FrameReader(handler.awaitDataFrame()); reader.longTextTable("events", 121 + target, 131, target, Long.toString(Long.MAX_VALUE)); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } } }); @@ -1508,16 +1639,21 @@ public void testLongSymbolGenerationRemainsPinnedAcrossTextRebind() throws Excep sender.table("events"); try { sender.uuidColumn("value", 1, 2); - Assert.fail("expected target-changing schema refresh"); + Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.longColumn("value", 8).atNow(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 8).atNow(); sender.flush(); - new FrameReader(handler.awaitDataFrame()) + new FrameReader(handler.awaitDataFrames(2)) .twoSymbolAndVarcharBlocks("events", 151, 161, 162); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -1548,7 +1684,8 @@ public void testAsymmetricUuidRowsArePinnedAndInvalidMiddleRowRollsBack() throws Assert.assertEquals(0x2122232425262728L, reader.i64()); Assert.assertEquals(0x3132333435363738L, reader.i64()); reader.eof(); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } }); } @@ -1605,31 +1742,38 @@ public void testMissingTableInfersColumnsAndUsesCachedAbsence() throws Exception } @Test - public void testKnownAdoptionRetainsPendingUnknownGeneration() throws Exception { + public void testKnownAdoptionWaitsForBatchBoundary() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, 31, 41); + handler.isSchemaFeedbackEnabled = true; try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").longColumn("id", 11).atNow(); handler.result = QwpSchemaProtocol.RESULT_KNOWN; + // The pending batch keeps its inferred snapshot: a conflicting input + // fails against the inferred LONG and the failed row rolls back whole. try { sender.stringColumn("failed_b", "partial"); sender.stringColumn("id", "11121314-1516-1718-0102-030405060708"); - Assert.fail("expected schema adoption"); + Assert.fail("expected inferred type conflict"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_CHANGED, e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.stringColumn("id", "21222324-2526-2728-3132-333435363738").atNow(); + Assert.assertEquals(1, handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("id", "21222324-2526-2728-3132-333435363738").atNow(); sender.flush(); - FrameReader reader = new FrameReader(handler.awaitDataFrame()); + FrameReader reader = new FrameReader(handler.awaitDataFrames(2)); reader.twoInferredAdoptionBlocks("events"); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @Test - public void testUnchangedMissingColumnRefreshesOnceAndRetainsInferredRows() throws Exception { + public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); try (TestWebSocketServer server = schemaServer(handler); @@ -1649,13 +1793,14 @@ public void testUnchangedMissingColumnRefreshesOnceAndRetainsInferredRows() thro Assert.assertEquals(1, reader.i64()); Assert.assertEquals(3, reader.i64()); reader.eof(); - Assert.assertEquals(2, handler.describeRequests.get()); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); } }); } @Test - public void testFreshLookupFailuresRemoveOnlyPartialInferredRow() throws Exception { + public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() throws Exception { assertMemoryLeak(() -> { int[] results = { QwpSchemaProtocol.RESULT_DENIED, @@ -1669,26 +1814,50 @@ public void testFreshLookupFailuresRemoveOnlyPartialInferredRow() throws Excepti }; for (int i = 0; i < results.length; i++) { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + handler.isSchemaFeedbackEnabled = true; try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").longColumn("id", 1).atNow(); handler.result = results[i]; + // The pending batch never consults the server: the conflict is + // local and only the partial row rolls back. try { sender.stringColumn("failed_only", "partial"); sender.stringColumn("id", "different inferred type"); - Assert.fail("expected fresh lookup failure"); + Assert.fail("expected inferred type conflict"); } catch (LineSenderSchemaException e) { - Assert.assertEquals(reasons[i], e.getReason()); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - handler.result = QwpSchemaProtocol.RESULT_MISSING; sender.longColumn("id", 3).atNow(); - sender.flush(); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); FrameReader reader = new FrameReader(handler.awaitDataFrame()); reader.schemaTable("events", -1, -1, 2, "id", QwpConstants.TYPE_LONG); Assert.assertEquals(0, reader.u8()); Assert.assertEquals(1, reader.i64()); Assert.assertEquals(3, reader.i64()); reader.eof(); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The server could not describe the table for the stale frame, so + // its ACK invalidated the cache. The next batch looks the table up + // again and reports the lookup failure from the setter. + sender.table("events"); + try { + sender.longColumn("id", 5); + Assert.fail("expected lookup failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reasons[i], e.getReason()); + } + Assert.assertEquals(2, handler.describeRequests.get()); + handler.result = QwpSchemaProtocol.RESULT_MISSING; + sender.table("events").longColumn("id", 5).atNow(); + sender.flush(); + reader = new FrameReader(handler.awaitDataFrames(2).get(1)); + reader.schemaTable("events", -1, -1, 1, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(5, reader.i64()); + reader.eof(); Assert.assertEquals(3, handler.describeRequests.get()); } } @@ -1830,9 +1999,10 @@ public void testOldPeerUsesLegacyGeoHashWireAndNeverDescribes() throws Exception } @Test - public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() throws Exception { + public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 51, 52); + handler.isSchemaFeedbackEnabled = true; try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").uuidColumn("id", 1, 2).atNow(); @@ -1840,15 +2010,24 @@ public void testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows() sender.table("events"); try { sender.stringColumn("id", "still-invalid"); - Assert.fail("expected unchanged UUID validation error"); + Assert.fail("expected UUID validation error"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + // The rejection neither looks up metadata nor splits the batch: the + // next row joins the block pinned to version 52. sender.uuidColumn("id", 3, 4).atNow(); + long fsn = sender.flushAndGetSequence(); + new FrameReader(handler.awaitDataFrame()).uuidTable("events", 51, 52, 1, 2, 3, 4); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The stale frame's ACK carries version 53; the next batch adopts it. + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); sender.flush(); - byte[] frame = handler.awaitDataFrame(); - new FrameReader(frame).twoUuidBlocks("events", 51, 52, 1, 2, 53, 3, 4); - Assert.assertEquals(2, handler.describeRequests.get()); + new FrameReader(handler.awaitDataFrames(2).get(1)).uuidTable("events", 51, 53, 5, 6); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); } }); } @@ -2111,9 +2290,10 @@ public void testAsyncRecoveredSchemaBacklogDoesNotFallBackToLegacy() throws Exce } @Test - public void testResetDiscardsRetiredSchemaGeneration() throws Exception { + public void testResetDiscardsRowsAndKeepsCachedSnapshot() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 101, 102); + handler.isSchemaFeedbackEnabled = true; try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").uuidColumn("id", 1, 2).atNow(); @@ -2125,15 +2305,23 @@ public void testResetDiscardsRetiredSchemaGeneration() throws Exception { } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + // reset() discards the rows without sending anything, so no ACK can + // deliver version 103: the next batch still pins the cached 102. sender.reset(); sender.table("events").uuidColumn("id", 3, 4).atNow(); - sender.flush(); + long fsn = sender.flushAndGetSequence(); FrameReader reader = new FrameReader(handler.awaitDataFrame()); - reader.schemaTable("events", 101, 103, 1, "id", QwpConstants.TYPE_UUID); + reader.schemaTable("events", 101, 102, 1, "id", QwpConstants.TYPE_UUID); Assert.assertEquals(0, reader.u8()); Assert.assertEquals(3, reader.i64()); Assert.assertEquals(4, reader.i64()); reader.eof(); + // That frame's ACK carries 103 for the batch after it. + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2).get(1)).uuidTable("events", 101, 103, 5, 6); + Assert.assertEquals(1, handler.describeRequests.get()); } }); } @@ -2386,11 +2574,76 @@ private int targetType() { } } + /** + * Builds the cumulative ACK the fake servers send for a data frame. Like the real + * server, a handler attaches the current schema of {@code tableName} when the + * snapshot it last delivered to the client is stale, so the client learns about + * a change on the batch boundary without a DESCRIBE of its own. + */ + private static byte[] ackWithSchemaUpdate(long sequence, String tableName, byte[] schemaPayload) { + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + ByteBuffer ack = ByteBuffer.allocate(1 + 8 + 2 + 2 + 2 + name.length + 4 + schemaPayload.length) + .order(ByteOrder.LITTLE_ENDIAN); + ack.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(sequence).putShort((short) 0) + .putShort((short) 1).putShort((short) name.length).put(name) + .putInt(schemaPayload.length).put(schemaPayload); + return ack.array(); + } + + /** The ACK a server sends when it cannot produce the schema snapshot a stale frame calls for. */ + private static byte[] ackInvalidateAll(long sequence) { + ByteBuffer ack = ByteBuffer.allocate(1 + 8 + 2).order(ByteOrder.LITTLE_ENDIAN); + ack.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)) + .putLong(sequence).putShort((short) 0); + return ack.array(); + } + + private static byte[] schemaControlFrame(byte[] schemaPayload) { + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + schemaPayload.length) + .order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(schemaPayload.length) + .put(schemaPayload); + return out.array(); + } + + /** + * Walks one QWP data frame, or a sequence of single-table frames. A table + * pins one schema snapshot per batch, so two schema generations of the same + * table always arrive in two frames; the two-block helpers read the second + * block from the second frame, skipping its message header and symbol + * dictionary prelude. + */ private static final class FrameReader { - private final ByteBuffer in; + private final List frames; + private int frameIndex; + private ByteBuffer in; private FrameReader(byte[] data) { - in = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + this(Collections.singletonList(data)); + } + + private FrameReader(List frames) { + this.frames = frames; + this.in = ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN); + } + + private void nextFrameIfExhausted() { + if (in.hasRemaining() || frameIndex + 1 >= frames.size()) { + return; + } + frameIndex++; + in = ByteBuffer.wrap(frames.get(frameIndex)).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + varint(); + for (int i = 0, n = varint(); i < n; i++) { + string(); + } } private long i64() { @@ -3289,7 +3542,13 @@ private void messageHeader(int tableCount) { Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); Assert.assertEquals(QwpConstants.VERSION, u8()); Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); - Assert.assertEquals(tableCount, in.getShort() & 0xffff); + if (frames.size() == 1) { + Assert.assertEquals(tableCount, in.getShort() & 0xffff); + } else { + // One generation per frame: the blocks span the frame sequence. + Assert.assertEquals(tableCount, frames.size()); + Assert.assertEquals(1, in.getShort() & 0xffff); + } Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); } @@ -3301,6 +3560,7 @@ private void schemaBlockHeader( String column, byte wireType ) { + nextFrameIfExhausted(); Assert.assertEquals(table, string()); Assert.assertEquals(1, u8()); Assert.assertEquals(tableId, in.getInt()); @@ -3420,6 +3680,7 @@ private String stringBytes(int length) { } private void eof() { + Assert.assertEquals("unread frames", frames.size() - 1, frameIndex); Assert.assertFalse("unexpected trailing column or value bytes", in.hasRemaining()); } @@ -3453,34 +3714,20 @@ private void schemaTable(String table, int tableId, long version, int rows, Stri Assert.assertEquals(type & 0xff, u8()); } - private void twoUuidBlocks( - String table, - int tableId, - long firstVersion, - long firstLo, - long firstHi, - long secondVersion, - long secondLo, - long secondHi - ) { - Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); - Assert.assertEquals(QwpConstants.VERSION, u8()); - Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); - Assert.assertEquals(2, in.getShort() & 0xffff); - Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + private void uuidTable(String table, int tableId, long version, long... loHiPairs) { + messageHeader(1); Assert.assertEquals(0, varint()); Assert.assertEquals(0, varint()); - uuidBlock(table, tableId, firstVersion, firstLo, firstHi); - uuidBlock(table, tableId, secondVersion, secondLo, secondHi); + schemaBlockHeader(table, tableId, version, loHiPairs.length / 2, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + for (long word : loHiPairs) { + Assert.assertEquals(word, i64()); + } eof(); } private void twoInferredAdoptionBlocks(String table) { - Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); - Assert.assertEquals(QwpConstants.VERSION, u8()); - Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); - Assert.assertEquals(2, in.getShort() & 0xffff); - Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + messageHeader(2); Assert.assertEquals(0, varint()); Assert.assertEquals(0, varint()); @@ -3493,22 +3740,11 @@ private void twoInferredAdoptionBlocks(String table) { Assert.assertEquals(0, u8()); Assert.assertEquals(11, i64()); - uuidBlock(table, 31, 41, 0x3132333435363738L, 0x2122232425262728L); - eof(); - } - - private void uuidBlock(String table, int tableId, long version, long lo, long hi) { - Assert.assertEquals(table, string()); - Assert.assertEquals(1, u8()); - Assert.assertEquals(tableId, in.getInt()); - Assert.assertEquals(version, in.getLong()); - Assert.assertEquals(1, varint()); - Assert.assertEquals(1, varint()); - Assert.assertEquals("id", string()); - Assert.assertEquals(QwpConstants.TYPE_UUID, u8()); + schemaBlockHeader(table, 31, 41, 1, "id", QwpConstants.TYPE_UUID); Assert.assertEquals(0, u8()); - Assert.assertEquals(lo, i64()); - Assert.assertEquals(hi, i64()); + Assert.assertEquals(0x3132333435363738L, i64()); + Assert.assertEquals(0x2122232425262728L, i64()); + eof(); } private void uuidOnlyTable(String table, int tableId, long version, long lo, long hi) { @@ -3547,6 +3783,12 @@ private static final class SchemaHandler implements TestWebSocketServer.WebSocke private final int tableId; private volatile long version; private long ackSequence; + // The snapshot the client last received for "events", through DESCRIBE or ACK + // feedback. With feedback enabled, a data frame acknowledged while that + // snapshot is stale carries the update, as the real server does. + private int deliveredResult = -1; + private long deliveredVersion = -1; + private volatile boolean isSchemaFeedbackEnabled; private SchemaHandler(int result, int tableId, long version) { this(result, tableId, version, true); @@ -3584,8 +3826,13 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien byte[] name = new byte[nameLength]; request.position(QwpConstants.HEADER_SIZE + 1 + Long.BYTES + Short.BYTES); request.get(name); - int responseTableId = tableId + ("b".equals(new String(name, StandardCharsets.UTF_8)) ? 1 : 0); - sendSchema(client, requestId, responseTableId); + String tableName = new String(name, StandardCharsets.UTF_8); + int responseTableId = tableId + ("b".equals(tableName) ? 1 : 0); + if ("events".equals(tableName)) { + deliveredResult = result; + deliveredVersion = version; + } + send(client, schemaControlFrame(schemaPayload(requestId, responseTableId))); return; } dataFrames.add(data); @@ -3593,8 +3840,25 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (!acknowledgeData) { return; } + send(client, ack()); + } + + private byte[] ack() { + long sequence = ackSequence++; + if (!isSchemaFeedbackEnabled || (result == deliveredResult && version == deliveredVersion)) { + return QwpWireTestUtils.buildAck(sequence); + } + if (result != QwpSchemaProtocol.RESULT_KNOWN && result != QwpSchemaProtocol.RESULT_MISSING) { + return ackInvalidateAll(sequence); + } + deliveredResult = result; + deliveredVersion = version; + return ackWithSchemaUpdate(sequence, "events", schemaPayload(0, tableId)); + } + + private static void send(TestWebSocketServer.ClientHandler client, byte[] frame) { try { - client.sendBinary(QwpWireTestUtils.buildAck(ackSequence++)); + client.sendBinary(frame); } catch (IOException e) { throw new AssertionError(e); } @@ -3644,7 +3908,7 @@ private synchronized List awaitDataFrames(int count) throws InterruptedE return new ArrayList<>(dataFrames); } - private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId, int responseTableId) { + private byte[] schemaPayload(long requestId, int responseTableId) { byte[] id = "id".getBytes(StandardCharsets.UTF_8); byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); @@ -3655,22 +3919,15 @@ private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId + 2 + failed.length + 4 + 2 + 2 + ts.length + 4 + 2; } - ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) - .order(ByteOrder.LITTLE_ENDIAN); - out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) - .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payloadLength) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + ByteBuffer out = ByteBuffer.allocate(payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); if (result == QwpSchemaProtocol.RESULT_KNOWN) { out.putInt(responseTableId).putLong(version).putShort((short) designatedIndex).putShort((short) 3) .putShort((short) id.length).put(id).putInt(ColumnType.UUID).putShort((short) 0) .putShort((short) failed.length).put(failed).putInt(ColumnType.STRING).putShort((short) 0) .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); } - try { - client.sendBinary(out.array()); - } catch (IOException e) { - throw new AssertionError(e); - } + return out.array(); } } @@ -3681,6 +3938,8 @@ private static final class LongTextSchemaHandler implements TestWebSocketServer. private volatile int targetType; private volatile long version; private long ackSequence; + // The version the client last received, through DESCRIBE or ACK feedback. + private long deliveredVersion = -1; private LongTextSchemaHandler(int tableId, long version, int targetType) { this.tableId = tableId; @@ -3695,13 +3954,24 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien describeRequests.incrementAndGet(); ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); - sendSchema(client, requestId); + deliveredVersion = version; + send(client, schemaControlFrame(schemaPayload(requestId))); return; } dataFrames.add(data); notifyAll(); + long sequence = ackSequence++; + if (version == deliveredVersion) { + send(client, QwpWireTestUtils.buildAck(sequence)); + } else { + deliveredVersion = version; + send(client, ackWithSchemaUpdate(sequence, "events", schemaPayload(0))); + } + } + + private static void send(TestWebSocketServer.ClientHandler client, byte[] frame) { try { - client.sendBinary(QwpWireTestUtils.buildAck(ackSequence++)); + client.sendBinary(frame); } catch (IOException e) { throw new AssertionError(e); } @@ -3719,7 +3989,19 @@ private synchronized byte[] awaitDataFrame() throws InterruptedException { return dataFrames.get(0); } - private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId) { + private synchronized List awaitDataFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.size() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for " + count + " data frames; received " + dataFrames.size()); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return new ArrayList<>(dataFrames); + } + + private byte[] schemaPayload(long requestId) { byte[] value = "value".getBytes(StandardCharsets.UTF_8); byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); @@ -3727,21 +4009,14 @@ private void sendSchema(TestWebSocketServer.ClientHandler client, long requestId + 2 + value.length + 4 + 2 + 2 + failed.length + 4 + 2 + 2 + ts.length + 4 + 2; - ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) - .order(ByteOrder.LITTLE_ENDIAN); - out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) - .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(payloadLength) - .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId) + ByteBuffer out = ByteBuffer.allocate(payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId) .put((byte) QwpSchemaProtocol.RESULT_KNOWN) .putInt(tableId).putLong(version).putShort((short) 2).putShort((short) 3) .putShort((short) value.length).put(value).putInt(targetType).putShort((short) 0) .putShort((short) failed.length).put(failed).putInt(ColumnType.UUID).putShort((short) 0) .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); - try { - client.sendBinary(out.array()); - } catch (IOException e) { - throw new AssertionError(e); - } + return out.array(); } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java index 3c46655cb..bbfea31ba 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java @@ -144,54 +144,6 @@ public void testGetColumnIndexIsCaseInsensitive() { Assert.assertEquals(-1, missing.getColumnIndex("id")); } - @Test - public void testHasSameRelevantTarget() { - byte[] none = new byte[0]; - QwpSchemaResponse base = decode(knownResponse(1, 1, 1, 1, - column("v", 5, none), column("ts", 10, none))); - - // Position and unrelated columns do not matter; name case does not either. - QwpSchemaResponse moved = decode(knownResponse(2, 1, 2, 0, - column("TS", 10, none), column("extra", 7, none), column("V", 5, none))); - Assert.assertTrue(base.hasSameRelevantTarget(moved, "v")); - Assert.assertTrue(base.hasSameRelevantTarget(moved, "ts")); - Assert.assertTrue(base.hasSameRelevantTarget(moved, null)); - Assert.assertTrue(moved.hasSameRelevantTarget(base, "V")); - - // Absent on both sides is the same target; absent on one side is not. - Assert.assertTrue(base.hasSameRelevantTarget(moved, "absent")); - Assert.assertFalse(base.hasSameRelevantTarget(moved, "extra")); - Assert.assertFalse(moved.hasSameRelevantTarget(base, "extra")); - - QwpSchemaResponse retyped = decode(knownResponse(3, 1, 3, 1, - column("v", 6, none), column("ts", 10, none))); - Assert.assertFalse(base.hasSameRelevantTarget(retyped, "v")); - Assert.assertTrue(base.hasSameRelevantTarget(retyped, "ts")); - - QwpSchemaResponse parameterized = decode(knownResponse(4, 1, 4, 1, - column("v", 5, new byte[]{1}), column("ts", 10, none))); - Assert.assertFalse(base.hasSameRelevantTarget(parameterized, "v")); - - // The designated role moves from "ts" to "v": both named targets change, - // and so does the designated timestamp's type. - QwpSchemaResponse redesignated = decode(knownResponse(5, 1, 5, 0, - column("v", 5, none), column("ts", 10, none))); - Assert.assertFalse(base.hasSameRelevantTarget(redesignated, "v")); - Assert.assertFalse(base.hasSameRelevantTarget(redesignated, "ts")); - Assert.assertFalse(base.hasSameRelevantTarget(redesignated, null)); - - QwpSchemaResponse undesignated = decode(knownResponse(6, 1, 6, -1, - column("v", 5, none), column("ts", 10, none))); - Assert.assertFalse(base.hasSameRelevantTarget(undesignated, null)); - Assert.assertTrue(undesignated.hasSameRelevantTarget(undesignated, null)); - Assert.assertTrue(base.hasSameRelevantTarget(undesignated, "v")); - - QwpSchemaResponse missing = decode(resultResponse(7, QwpSchemaProtocol.RESULT_MISSING)); - Assert.assertFalse(base.hasSameRelevantTarget(missing, "v")); - Assert.assertTrue(missing.hasSameRelevantTarget(missing, "v")); - Assert.assertTrue(missing.hasSameRelevantTarget(missing, null)); - } - @Test public void testRejectsMalformedEnvelopesAndTrailingData() { byte[] valid = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); diff --git a/design/schema-aware-sender-contract-simplification.md b/design/schema-aware-sender-contract-simplification.md new file mode 100644 index 000000000..8ba8b03e1 --- /dev/null +++ b/design/schema-aware-sender-contract-simplification.md @@ -0,0 +1,221 @@ +# Schema-aware Java sender: contract simplification proposals + +Date: 2026-09-18 + +Status: proposal 1 is implemented (see the "One schema snapshot per table per batch" section of +[schema-aware-sender.md](schema-aware-sender.md)), with one deviation: there is no explicit refresh +operation. Batch-boundary adoption relies on ACK schema feedback and reconnects, because a caller cannot +distinguish a stale-snapshot rejection from invalid input and so has no sound rule for when to refresh. +Proposals 2 to 4 remain proposals for discussion, not approved contract or implemented behavior. + +These ideas come from the second complexity review of client +`7a73e58e8d9354a045c9895c563e8009cd6ff8cf`, including the corresponding server paths inspected at +`6b0c992b18`. They deliberately question the contract rather than only refactoring its implementation. + +## Recommendation + +Simplify the promises before doing another mechanical cleanup. Much of the remaining complexity implements +server-compatible casting, automatic schema refresh, and adaptation while batches are being built. Those promises +are stronger than the core requirement: + +> Validate types before creating, and potentially persisting, a data frame. + +The proposed direction is typed writes checked against a stable schema snapshot, followed by the existing encoder +and persistence pipeline. Do not build a general conversion framework or another buffering lifecycle. + +Start with proposal 1: extend the batch-boundary rule to schema versions and make refresh explicit. It offers a +substantial lifecycle simplification without first redesigning the conversion catalogue or wire protocol. + +Validation remains relative to the chosen snapshot. It cannot guarantee eventual server acceptance after concurrent +DDL, permission changes, or other server-side failures. The current implementation does not provide that guarantee +either. + +## 1. One schema snapshot per table per batch + +### Assumption to change + +Every new row should observe the latest cached schema, and a failed setter should automatically refresh metadata +before deciding which error to report. + +Currently, a schema identity change can replace a table buffer and retain its previous generation even when the +relevant column is unchanged. This requires pending generations, additional flush collection and accounting, and +cleanup across reset, flush, and close. The one-refresh policy also tracks whether each failing operation already +obtained fresh metadata and compares its old and new targets. + +### Proposed contract + +- Pin one schema snapshot per table for the entire pending batch. +- Adopt another snapshot only after successful flush or explicit reset. A failed flush retains the existing snapshot. +- Make explicit refresh an empty-batch operation. Initial lookup and lookup on a cache miss still obtain metadata. +- Report local validation failures directly; do not perform a network lookup from the failed setter. +- Do not implicitly flush pending rows merely to adopt a new schema. + +This is separate from the already-resolved legacy-to-modern transition. It applies the same boundary principle to +different schema versions within schema mode. + +### Complexity that can disappear + +`retiredTableBuffers`, multiple pending generations of one table, their flush/accounting/cleanup paths, per-operation +freshness bookkeeping, relevant-target comparison, and the automatic `SCHEMA_CHANGED` recovery path. + +Layout replacement is still necessary when a schema changes: `QwpTableBuffer.reset()` retains column definitions. +The simplification is that replacement happens when the old buffer has no pending rows, rather than retaining both +layouts for a later flush. + +### User impact + +Applications must coordinate schema changes and refresh deliberately. An input made valid by DDL may continue to +be rejected until refresh. Earlier completed rows remain intact; refresh does not silently discard or rewrite them. + +Evidence: [QwpWebSocketSender](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java), +especially `bindingForEffectiveWrite()`, `installSchemaBinding()`, `refreshAfterSchemaRejection()`, and +`collectNonEmptyTables()`. The existing +[`testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows()`](../core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java) +demonstrates separate generations after an unrelated version change. + +## 2. Keep schema identities out of new persisted data frames + +### Assumption to change + +Local schema validation requires a schema-extended data-frame format and schema feedback in write responses. + +The current `(tableId, metadataVersion)` pair is not an acceptance precondition. The server processes wire values +normally; matching identities suppress schema feedback. A version mismatch alone does not reject the write or +prevent the server from applying another conversion. + +### Proposed contract + +Retain schema lookup and local validation, then emit ordinary QWP data frames. Replace automatic ACK/NACK schema +feedback with explicit refresh. Keep the validation snapshot in memory; replay does not need to reconstruct it. + +This requires a coordinated client/server change, not merely ignoring feedback in the client. Schema lookup still +needs capability negotiation and authorization. + +### Complexity that can disappear + +- Schema identities and the additional data-frame encoding path. +- Schema-update lists and invalidate-all responses, including their ordering and size-limit rules. +- Schema-specific persisted-frame detection and replay capability restrictions for the new ordinary frames. + +### User impact and migration constraints + +Applications lose passive schema updates from writes and must refresh deliberately. Ordinary ACK, durable ACK, +terminal-NACK, and replay semantics must remain unchanged. + +Do not equate ordinary framing with reverting all value encoding. In particular, preserve the forced GEOHASH null +bitmap needed to distinguish valid all-one values from sentinel NULL. Calling today's legacy encoder unchanged is +not an equivalent implementation. + +Any already-persisted extended frames must retain a compatible reader/drainer until they are drained. Do not strip +identities, rewrite queued frames, discard data, or advance watermarks to simplify migration. That compatibility +requirement may delay deletion of old replay support. + +Evidence: server +[`QwpIngressProcessorState.captureSchemaFeedback()`](../../core/src/main/java/io/questdb/cutlass/qwp/server/QwpIngressProcessorState.java); +client [QwpWebSocketEncoder](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java), +[WebSocketResponse](../core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java), and +[`QwpColumnWriter.encodeSchemaTable()`](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java). + +## 3. Validate typed inputs instead of reproducing the server's casting catalogue + +### Assumption to change + +The client must accept essentially every intentional setter-to-column conversion that the server accepts, with +compatible parsing, formatting, rounding, and null behavior. + +`stringColumn()` currently supports numeric parsing, timestamps, dates, UUIDs, decimals, geohashes, binary, and +other targets. Other setters implement reverse formatting conversions. This makes the binding a conversion engine, +not merely a validator. + +### Proposed contract + +Accept native typed inputs and a small, explicitly documented set of safe conversions. Reject arbitrary cross-family +casts before framing. For example: + +- UUID columns require UUID input rather than UUID-looking strings. +- Timestamp columns use timestamp setters. +- Applications format numbers themselves when writing text columns. +- DOUBLE arrays remain the only supported array element type; validate the rank against the target. + +Keep value-dependent validation where type compatibility alone is insufficient: decimal precision and scale, +timestamp overflow, numeric range where conversions remain, and array shape/rank constraints. Narrowing the cast +catalogue must not defer these checks to the server. + +### Complexity that can disappear + +Temporal text conversion alone accounts for the dedicated 818-line parser and 187-line formatter at the reviewed +revision. Removing those conversions makes both files unnecessary. A narrower catalogue also removes many binding +switch branches, formatting/parsing helpers, scratch sinks, and compatibility-specific test cases. + +Do not delete shared utilities merely because one consumer disappears. The existing typed APIs may still need +parsing or representation normalization; the target is implicit cross-family casting, not all conversion code. + +### User impact + +Some currently accepted setter/column combinations become local errors. Applications must parse or format explicitly +and choose the appropriate setter. DATE needs a deliberate mapping because `Sender` has no `dateColumn()` method. + +This is the largest compatibility change, but also the largest reduction in conversion code. Agree on the retained +conversion list before implementation rather than growing it opportunistically during the rewrite. + +Evidence: [QwpSchemaBinding](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java), +[QwpSchemaTimestampParser](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java), +and [QwpSchemaTimestampFormatter](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java). + +## 4. Require a known target schema for validated writes + +### Assumption to change + +The same validated path must also support old servers, fresh-process offline ingestion, and automatic table/column +creation without known target metadata. + +The current asynchronous cold-start path deliberately accepts legacy writes without metadata. It cannot satisfy an +unconditional validate-before-framing guarantee. A confirmed missing table or column likewise has no existing target +type to validate against; inference is a separate contract. + +### Proposed contract + +- Cold-start writes wait or fail until metadata is available. +- Cached schemas may still support offline buffering. +- Unknown tables and columns require provisioning and refresh instead of inference and automatic creation. +- If old-server compatibility remains, expose it as explicitly unchecked behavior outside the validated contract, + not as an automatic fallback within it. + +### Complexity that can disappear + +Missing-schema inference and adoption, unknown target identities, and opportunistic legacy-to-schema transitions +within the validated path. + +### User impact + +Less zero-setup ingestion, no fresh-process offline ingestion without metadata, and explicit schema provisioning. +Do not compensate automatically by adding a persisted metadata cache: that introduces another format and freshness +lifecycle that needs its own justification. + +Evidence: [`QwpWebSocketSender.bindingForEffectiveWrite()`](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java), +the missing-target branches in [QwpSchemaBinding](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java), +and `testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect()` in +[QwpSchemaSenderIntegrationTest](../core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java). + +## Validation timing and complexity worth keeping + +The core requirement permits type errors at `at()`/`atNow()` rather than at every setter. Allow that flexibility if it +lets validation reuse the existing row-commit and rollback machinery. Do not introduce a second store of original +inputs merely to delay validation. + +Avoid moving all validation to flush: recovering one rejected row is substantially easier than reconstructing a +rejected batch. Every encoding path, including auto-flush and close, must still be protected before it creates a +data frame. + +Keep immutable snapshots, bounded lookup with cancellation, authorization checks, row rollback, parameter/range +validation, and unchanged-byte replay. Those mechanisms support the actual requirement or existing delivery safety. + +## Review validation and limits + +The review ran 252 focused client tests on Java 8 at the reviewed revision; all passed. That validates the current +contract, not these proposed replacements. The proposals were derived from source and contract analysis, not an +implemented prototype or measured performance comparison. + +Each adopted contract change needs explicit regression coverage for its new acceptance/error behavior, pending-row +retention, auto-flush, transaction boundaries, and replay compatibility as applicable. No production implementation +was changed as part of this review or this document. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index ebea5e03d..abc8ef0cb 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -1453,8 +1453,7 @@ decide recovery. The proposed reason codes and actions are: | Reason | Caller action | |---|---| -| `INVALID_VALUE` | Correct or reject the input; reconstruct the whole row if writing it again. Retrying the same input against the same target will fail. | -| `SCHEMA_CHANGED` | Reconstruct and retry the whole row using the refreshed schema. | +| `INVALID_VALUE` | Correct or reject the input; reconstruct the whole row if writing it again. Retrying the same input against the same snapshot will fail. If server DDL has since made the input valid, the next batch after an acknowledged flush validates against the new schema. | | `SCHEMA_UNAVAILABLE` | Retry the whole row when metadata can be obtained, for example after a timeout or temporary outage. | | `ACCESS_DENIED` | Resolve authorization; unchanged credentials/permissions do not make an immediate retry useful. | | `UNSUPPORTED_FEATURE` | Use a supporting endpoint/client for an extension required after upgrade or by pending extended SF, or for an unsupported target type. Do not downgrade or fall back to raw encoding. | @@ -1470,79 +1469,67 @@ recovery. Existing transport/flush failures and terminal server NACKs keep their own delivery and recovery semantics. Test the reason, row outcome and allowed next public API operation, not only the diagnostic text. -A cached schema must not reject indefinitely after DDL makes an input valid. -Before reporting a conversion rejection against cached metadata, describe once. -If this operation already obtained a fresh schema, do not look up again. -Freshness belongs to the failing setter, not the whole row: a describe performed -by an earlier setter does not suppress refresh for a later failing setter. - -- If the relevant target is unchanged, preserve the original rejection reason: - `INVALID_VALUE` for an invalid value or `UNSUPPORTED_FEATURE` for an unsupported - conversion. A changed metadata version alone is not `SCHEMA_CHANGED`. -- If the target or table incarnation changed, cancel the row and report - `SCHEMA_CHANGED`. Prepare the fresh snapshot for the next whole-row retry. -- If refresh fails, cancel the partial row and report its actual reason: - temporary unavailability, access denial or unsupported functionality. - -Do not retry only the failing setter using the fresh schema: earlier values -in the same row have already been converted under the old one. The sender -does not retain original inputs for automatic whole-row reconversion. -There is at most one refresh per failing operation, not a DDL retry loop. -An implemented input family rejected against a cached missing column or target -type also gets this refresh, so a newly added or changed server column can be -discovered. An intrinsically unimplemented input family fails directly; looking -up another schema cannot implement that setter. Lookup and authorization errors -are not conversion-retry triggers. - -## Pinned blocks and schema transitions +A conversion rejection is final for the snapshot the batch pinned. The failed +setter reports `INVALID_VALUE` or `UNSUPPORTED_FEATURE` directly; it never +performs a network lookup, so a rejection costs no round trip and cannot fail +for a transport reason. A cached schema therefore keeps rejecting an input that +DDL has since made valid until the batch boundary brings the new snapshot (see +the next section). Lookup and authorization errors surface only from the path +that actually looks metadata up: the first row of a batch on a cache miss. + +Do not retry only the failing setter after a refresh: earlier values in the +same row have already been converted under the old snapshot. The sender does +not retain original inputs for automatic whole-row reconversion. + +## One schema snapshot per table per batch Keep two concepts separate: -- The latest server snapshot, learned through describes or ACKs/NACKs. -- The encoding snapshot pinned to a row and its table block. +- The latest server snapshot, learned through describes or ACK feedback. +- The encoding snapshot pinned to a table for the pending batch. -All rows in a schema-mode table block use one encoding snapshot. Its identity +All rows a table holds in one batch use one encoding snapshot. Its identity describes the schema used to resolve server-backed bindings, with explicit inferred bindings for any missing columns. It is not replaced by the latest cache identity when the block is sent. Wire column definitions still describe the actual bytes; the identity does not replace those definitions. -For example, an ACK for an earlier batch may arrive while another block is -being built: +The sender resolves the cache only when a table's buffer holds no rows: at the +first effective write of the table's first row in a batch, including +timestamp-only rows. From then on every row of that table in the batch reuses +the pinned binding without consulting the cache, so an ACK that lands mid-batch +with a newer snapshot cannot split the batch or relabel buffered values: ``` -Server v10: count INT -> current block: v10, INT, four-byte values -Server v11: count LONG -> next block: v11, LONG, eight-byte values +Server v10: count INT -> this batch: v10, INT, four-byte values +Server v11: count LONG -> next batch: v11, LONG, eight-byte values ``` -Publishing v11 cannot relabel existing four-byte values as LONG or start -appending eight-byte values to their column buffer. - -The I/O thread only publishes snapshots. The producer finishes an in-progress -row under its pinned snapshot, or cancels it on error. At the first effective -write of each schema-mode row, including timestamp-only rows, it resolves the -latest cached schema or performs a lookup on a miss. If the identity changed, -it seals completed rows under the old snapshot and installs a new layout before -accepting the new row's first value. This boundary is detected from actual row -state, including rows started without another `table()` call. Repeating -`table()` is not itself a safe transition point. - -For the first implementation, adopting a different identity starts a new -block. Reusing unchanged allocations is allowed, but not relabeling existing -rows. Avoiding splits for provably irrelevant updates is an optional later -optimization, not a reason to mix snapshots. - -Sealing freezes a block; it need not wait for a network send or ACK. Keep sealed -blocks ordered and included in normal pending-row/byte accounting, flush limits -and backpressure. Do not introduce an unbounded queue. Successive generations -of one table can use separate frames; preserve existing multi-table transaction -and deferred-commit boundaries. - -The current `QwpTableBuffer.reset()` preserves column definitions. Transition -therefore needs an explicit producer-owned layout replacement after old rows -have been retained for encoding or encoded. Invalidate column-definition and -designated-timestamp caches with that replacement. Never clear data still -owned by a pending block. +Adoption happens at the batch boundary. After a successful `flush()` or an +explicit `reset()` the table buffer is row-less again, and the next row +compares the pinned identity with the cache; a different identity replaces the +buffer's layout (`QwpTableBuffer.reset()` keeps column definitions, so the +buffer is cleared and rebound in place) before accepting the first value. A +failed flush retains the batch and its snapshot. + +There is no explicit refresh API. The cache stays current through the server: +a frame that carries a stale identity makes the server attach the table's +current schema to the ACK, and the send loop applies that feedback before it +advances the ack watermark, so an application that flushes and waits for the +acknowledgment before starting the next batch adopts the change deterministically. +When the server cannot produce the snapshot it answers `INVALIDATE_ALL`, the +cache empties, and the next batch's first row looks the table up again and +reports `SCHEMA_UNAVAILABLE`, `ACCESS_DENIED` or `UNSUPPORTED_FEATURE` from the +setter. A reconnect also clears the cache. A producer whose every new row is +rejected by a stale snapshot never ships a frame and therefore never receives +feedback; recovery from that state is a reconnect or a new sender. An explicit +refresh was considered and dropped: a caller cannot tell a stale-snapshot +rejection from invalid input, so there is no sound rule for when to call it. + +A frame therefore never carries two generations of one table, and the sender +keeps no retired buffers, pending-generation lists or per-operation freshness +bookkeeping. Persisted frames from before this rule may still hold two blocks +of one table; the replay path reads them unchanged. ## Schema feedback through writes @@ -2153,18 +2140,21 @@ without claiming general exactly-once delivery. fractions and `Instant` precision at both target resolutions. Include valid negative instants whose intermediate seconds multiplication would overflow, duplicate suppression, local failure/rollback and unchanged legacy behavior. -- A stale rejection refreshes once. A changed target cancels the whole row; - retry uses the new snapshot. No setter splices new rules into a partial row. +- A stale rejection is local: it issues no describe, cancels the whole row and + leaves the batch pinned. No setter splices new rules into a partial row. - Local errors expose the expected reason and recovery action. An invalid row - stays cancelled after flush; whole-row retries after schema refresh or a - recovered lookup can succeed without losing earlier completed rows. Access - denial and unsupported functionality are not mislabeled as transient lookup - failures or flush-retryable errors. -- ACK metadata arriving mid-row leaves that row and completed rows unchanged. - The next adopted snapshot gets a separate block with matching wire types - and identity; exercise INT-to-LONG and cached designated timestamps. -- Transitions work without another `table()` call, preserve pending accounting, - frame order, transaction boundaries, backpressure and completed rows on error. + stays cancelled after flush; whole-row retries in the next batch after ACK + feedback or a recovered lookup can succeed without losing earlier completed + rows. Access denial and unsupported functionality are not mislabeled as + transient lookup failures or flush-retryable errors, and an `INVALIDATE_ALL` + ACK makes the next batch's lookup report them from the setter. +- ACK metadata arriving mid-batch leaves that batch unchanged. The next batch + adopts the snapshot with matching wire types and identity in its own frame; + exercise INT-to-LONG and cached designated timestamps. +- `reset()` discards rows without shipping a frame, so it delivers no feedback; + the next batch pins the cached snapshot. Transitions preserve pending + accounting, frame order, transaction boundaries, backpressure and completed + rows on error. - Auto-create, multi-table writes, cumulative ACKs, deferred commits, delayed sends and describe/ACK interleaving preserve schema feedback and watermarks. - After schema confirmation, lookup timeout, offline cache misses, eviction, From 7d5f34e0635f67dfec8c7e04e6b87ae953ab3d13 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 18 Sep 2026 11:37:12 +0200 Subject: [PATCH 27/51] Move FLAG_SCHEMA from 0x02 to 0x40 Server PR #7531 (browser negotiation) already claimed header flag bit 0x02 for FLAG_DURABLE_ACK_POLL, a zero-table control frame. Merging server master into the schema-aware branch produced a clean textual merge, but the ingress dispatch then read a poll frame as a schema-flagged frame on a connection that had not negotiated schema and rejected it. The poll bit is already on server master, while FLAG_SCHEMA is unreleased, so FLAG_SCHEMA moves to the free bit 0x40. The server constant changes in lockstep. The design doc and the journal record the new value and the reason. Store-and-forward frames persisted by an earlier build of this branch carry 0x02 and no longer replay as schema frames. Co-Authored-By: Claude Fable 5.1 --- .../client/cutlass/qwp/protocol/QwpConstants.java | 2 +- design/schema-aware-sender-journal.md | 14 ++++++++++++++ design/schema-aware-sender.md | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index a99f7bf70..aac4289c8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -42,7 +42,7 @@ public final class QwpConstants { */ public static final byte FLAG_DEFER_COMMIT = 0x01; /** Flag bit: table headers carry schema identity information. */ - public static final byte FLAG_SCHEMA = 0x02; + public static final byte FLAG_SCHEMA = 0x40; /** * Flag bit: Delta symbol dictionary encoding enabled. * When set, symbol columns use global IDs and send only new dictionary entries. diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 9e40ffe97..322acecdd 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -8043,3 +8043,17 @@ from the column setter. `installSchemaBinding()` bound in place whenever the buffer had no rows, but a flushed legacy buffer keeps its column layout. It now binds in place only a buffer with no columns and replaces the buffer otherwise. Both rewritten upgrade tests cover that path. + +### FLAG_SCHEMA moves from 0x02 to 0x40 + +Merging server `master` on 2026-09-18 surfaced a wire-bit collision that git +merged cleanly: server PR #7531 (browser negotiation) had already claimed `0x02` +for `FLAG_DURABLE_ACK_POLL`, a zero-table control frame. The ingress dispatch +read a poll frame as a schema-flagged frame on a connection that had not +negotiated schema and rejected it, failing seven durable-ack poll tests in +`QwpIngressAckLeapfrogTest` and `QwpIngressUpgradeProcessorResumeRecvTest`. +The poll bit is already on `master`, while `FLAG_SCHEMA` is unreleased, so +`FLAG_SCHEMA` moves to the free bit `0x40` in both the server and the client +`QwpConstants`. This supersedes the `FLAG_SCHEMA=0x02` value recorded in design +revision 13. The trade-off: store-and-forward frames persisted by an earlier +build of this branch carry `0x02` and no longer replay as schema frames. diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index abc8ef0cb..6d75d47d4 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -779,7 +779,8 @@ it. This is the cost of choosing asynchronous startup. ## Frame layout and replay compatibility Negotiation authorizes the extension; a persisted frame flag identifies the -actual data layout. Use `FLAG_SCHEMA = 0x02` without changing the QWP version byte: +actual data layout. Use `FLAG_SCHEMA = 0x40` without changing the QWP version byte +(`0x02` belongs to the server's `FLAG_DURABLE_ACK_POLL`): - Flag set: each table header includes its pinned schema identity, or an explicit unknown identity. The server uses the extended table-header parser. From 89e15f0f180464ae065ad878f236677d4646f51e Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 18 Sep 2026 13:32:31 +0200 Subject: [PATCH 28/51] Keep schema lookups alive across reconnects Retain pending schema lookups when the transport disconnects, including requests that have not reached the wire. Resend on the replacement connection with a fresh request ID while preserving the original lookup deadline, so a brief server outage does not abort row construction. Keep explicit schema errors and malformed-response cancellation intact. Add lifecycle coverage for repeated disconnects, stale replies, unsent requests, timeout, interruption and close, and update the design contract. Validate 296 client tests on Java 8 and Java 25, the original failing fuzz seeds, and 18 server integration tests. All pass. --- .../sf/cursor/CursorWebSocketSendLoop.java | 3 +- .../sf/cursor/QwpSchemaCoordinator.java | 35 ++- .../CursorWebSocketSchemaLifecycleTest.java | 248 ++++++++++++++++-- design/schema-aware-sender.md | 31 ++- 4 files changed, 284 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 811ad2431..e440a6d8a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -4034,6 +4034,7 @@ public void onBinaryMessage(long payloadPtr, int payloadLen) { } requestId = QwpSchemaProtocol.peekResponseRequestId(payloadPtr, payloadLen); } catch (RuntimeException e) { + schemaCoordinator.invalidResponse(responseClient, 0); fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); return; } @@ -4044,7 +4045,7 @@ public void onBinaryMessage(long payloadPtr, int payloadLen) { try { schema = QwpSchemaProtocol.decodeResponse(payloadPtr, payloadLen); } catch (RuntimeException e) { - if (!schemaCoordinator.isLiveResponse(responseClient, requestId)) { + if (!schemaCoordinator.invalidResponse(responseClient, requestId)) { return; } fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index 367dd59eb..b1281caa6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -65,7 +65,7 @@ QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean re long id = nextRequestId; byte[] message = QwpSchemaProtocol.encodeDescribe(id, tableName); nextRequestId++; - Request own = new Request(id, key, startNanos, timeoutNanos, message); + Request own = new Request(id, key, tableName.toString(), startNanos, timeoutNanos, message); request = own; hasPendingRequest = true; LockSupport.unpark(ioThread); @@ -96,6 +96,14 @@ synchronized Request requestToSend(WebSocketClient client) { complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema lookup timed out")); return null; } + if (request.id == 0) { + if (nextRequestId <= 0 || nextRequestId == Long.MAX_VALUE) { + complete(request, null, failure(UNSUPPORTED_FEATURE, request.key, "schema request id space is exhausted")); + return null; + } + request.id = nextRequestId++; + request.message = QwpSchemaProtocol.encodeDescribe(request.id, request.tableName); + } request.sentClient = client; return request; } @@ -169,9 +177,24 @@ void applyFeedback(WebSocketResponse feedback) { } synchronized void connectionLost(WebSocketClient client) { + if (request != null && request.sentClient == client) { + // Keep the producer's request and deadline; only its wire attempt ended. + // A fresh ID also rejects old replies delivered on the replacement connection. + request.sentClient = null; + request.id = 0; + } + } + + synchronized boolean invalidResponse(WebSocketClient client, long requestId) { + // Zero means the malformed header could not identify a request. For an + // identifiable reply, cancellation must share the liveness check's lock. + if (requestId != 0 && !isLiveResponse(client, requestId)) { + return false; + } if (request != null && (request.sentClient == null || request.sentClient == client)) { - complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "connection lost during schema lookup")); + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "invalid schema control response")); } + return true; } synchronized void clearCache() { @@ -249,17 +272,19 @@ private void remove(String key) { static final class Request { final long startNanos; final long timeoutNanos; - final long id; + long id; final String key; - final byte[] message; + final String tableName; + byte[] message; boolean done; LineSenderSchemaException error; QwpSchemaResponse response; WebSocketClient sentClient; - Request(long id, String key, long startNanos, long timeoutNanos, byte[] message) { + Request(long id, String key, String tableName, long startNanos, long timeoutNanos, byte[] message) { this.id = id; this.key = key; + this.tableName = tableName; this.startNanos = startNanos; this.timeoutNanos = timeoutNanos; this.message = message; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java index 079941942..7a99876c2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -8,19 +8,25 @@ import io.questdb.client.LineSenderSchemaException; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cairo.ColumnType; +import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.runners.model.Statement; import java.io.File; import java.io.IOException; @@ -33,6 +39,23 @@ import java.util.concurrent.atomic.AtomicReference; public class CursorWebSocketSchemaLifecycleTest { + @Rule + public final TestRule memoryLeak = (base, description) -> new Statement() { + @Override + public void evaluate() throws Throwable { + AtomicReference failure = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + try { + base.evaluate(); + } catch (Throwable e) { + failure.set(e); + } + }); + if (failure.get() != null) { + throw failure.get(); + } + } + }; @Rule public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); @@ -90,7 +113,7 @@ public void testInterruptionRestoresFlagAndReleasesSlotForNextLookup() throws Ex } @Test - public void testDisconnectFailsCurrentLookupRetainsCacheAndQueuesMissForReplacement() throws Exception { + public void testDisconnectRetriesCurrentLookupRetainsCacheAndRejectsStaleReply() throws Exception { SchemaHandler handler = new SchemaHandler(); handler.reply = true; try (TestWebSocketServer server = server(handler); @@ -102,25 +125,22 @@ public void testDisconnectFailsCurrentLookupRetainsCacheAndQueuesMissForReplacem loop.start(); QwpSchemaResponse cached = loop.resolveSchema("cached", 5_000); handler.disconnectName = "lost"; + handler.disconnectsRemaining = 2; Lookup lost = lookup(loop, "lost", 5_000, null); Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); - lost.join(); - assertUnavailable(lost.error.get()); - Assert.assertSame(cached, loop.resolveSchema("CACHED", 5_000)); - CountDownLatch oneDone = new CountDownLatch(1); - Lookup first = lookup(loop, "fresh-one", 5_000, oneDone); - Lookup second = lookup(loop, "fresh-two", 5_000, oneDone); - Assert.assertTrue(oneDone.await(5, TimeUnit.SECONDS)); - reconnect.allow.countDown(); - first.join(); - second.join(); - Lookup successful = first.response.get() != null ? first : second; - Lookup busy = successful == first ? second : first; - Assert.assertNotNull(successful.response.get()); + Lookup busy = lookup(loop, "fresh", 5_000, null); + busy.join(); assertUnavailable(busy.error.get()); - Assert.assertSame(successful.response.get(), loop.resolveSchema( - handler.lastName.get(), 5_000)); + Assert.assertTrue(busy.error.get().getMessage().contains("another schema lookup")); + handler.sendStaleBeforeNextReply = true; + reconnect.allow.countDown(); + lost.join(); + Assert.assertNull(String.valueOf(lost.error.get()), lost.error.get()); + Assert.assertNotNull(lost.response.get()); + Assert.assertEquals(4, lost.response.get().getTableId()); + Assert.assertSame(lost.response.get(), loop.resolveSchema("lost", 0)); + Assert.assertEquals(3, server.handshakeCount()); int requestsBeforeCacheProbe = handler.requests; QwpSchemaResponse replacement = loop.resolveSchema("cached", 5_000); @@ -134,6 +154,158 @@ public void testDisconnectFailsCurrentLookupRetainsCacheAndQueuesMissForReplacem } } + @Test + public void testQueuedUnsentLookupSurvivesDisconnect() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("queued-disconnect"); + DisconnectingClient client = new DisconnectingClient()) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + reconnect.allow.countDown(); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Assert.assertTrue(client.receiving.await(5, TimeUnit.SECONDS)); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + client.disconnect.countDown(); + queued.join(); + Assert.assertNull(String.valueOf(queued.error.get()), queued.error.get()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, queued.response.get().getResult()); + Assert.assertEquals(1, handler.requests); + } finally { + client.disconnect.countDown(); + } + } + } + } + + @Test + public void testReconnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + handler.disconnectName = "deadline"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-deadline"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup lookup = lookup(loop, "deadline", 1_000, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + Assert.assertTrue(lookup.error.get().getMessage().contains("timed out")); + reconnect.allow.countDown(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, + loop.resolveSchema("probe", 5_000).getResult()); + Assert.assertEquals(2, handler.requests); + Assert.assertEquals("probe", handler.lastName.get()); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testInterruptDuringReconnectReleasesSlotForQueuedLookup() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + handler.disconnectName = "interrupted"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-interrupt"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup interrupted = lookup(loop, "interrupted", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + interrupted.thread.interrupt(); + interrupted.join(); + assertUnavailable(interrupted.error.get()); + Assert.assertTrue(interrupted.interrupted.get()); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + reconnect.allow.countDown(); + queued.join(); + Assert.assertNull(String.valueOf(queued.error.get()), queued.error.get()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, queued.response.get().getResult()); + Assert.assertEquals(2, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testCloseDuringReconnectReleasesLookupBeforeFactoryReturns() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.disconnectName = "closed"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-close"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + AtomicReference closeError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable e) { + closeError.set(e); + } + }); + try { + loop.start(); + Lookup lookup = lookup(loop, "closed", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + closer.start(); + lookup.join(); + assertUnavailable(lookup.error.get()); + Assert.assertTrue(lookup.error.get().getMessage().contains("closed")); + } finally { + reconnect.allow.countDown(); + closer.join(5_000); + Assert.assertFalse(closer.isAlive()); + } + Assert.assertNull(String.valueOf(closeError.get()), closeError.get()); + Assert.assertEquals(1, handler.requests); + } + } + } + + @Test + public void testMalformedCurrentReplyCancelsLookupEvenWithReconnectAvailable() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-malformed"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup lookup = lookup(loop, "malformed", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + handler.malformedReply = false; + handler.reply = true; + reconnect.allow.countDown(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, + loop.resolveSchema("probe", 5_000).getResult()); + Assert.assertEquals(2, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + @Test public void testAsyncInitialConnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { SchemaHandler handler = new SchemaHandler(); @@ -329,6 +501,15 @@ private static final class Lookup { private final AtomicReference response = new AtomicReference<>(); private Thread thread; + private void awaitWaiting() throws InterruptedException { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != Thread.State.TIMED_WAITING) { + Assert.assertTrue("lookup did not wait", thread.isAlive() && System.nanoTime() < deadline); + Thread.yield(); + } + } + private void join() throws InterruptedException { thread.join(5_000); Assert.assertFalse("lookup thread did not finish", thread.isAlive()); @@ -366,10 +547,42 @@ public void requireSchema() { } } + // Fault injection only: hold the receive poll until the producer has queued + // a lookup, then fail before the I/O thread can send that lookup. + private static final class DisconnectingClient extends WebSocketClient { + private final CountDownLatch disconnect = new CountDownLatch(1); + private final CountDownLatch receiving = new CountDownLatch(1); + + private DisconnectingClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + receiving.countDown(); + try { + Assert.assertTrue(disconnect.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + throw new LineSenderException("injected transport disconnect"); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { private final AtomicReference lastName = new AtomicReference<>(); private final CountDownLatch requestSeen = new CountDownLatch(1); private volatile String disconnectName; + private int disconnectsRemaining = 1; private volatile TestWebSocketServer.ClientHandler lastClient; private volatile long lastRequestId; private volatile boolean malformedReply; @@ -389,7 +602,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien lastRequestId = requestId; lastName.set(name); requestSeen.countDown(); - if (name.equals(disconnectName)) { + if (name.equals(disconnectName) && disconnectsRemaining-- > 0) { try { client.sendClose(1001, "test disconnect"); } catch (IOException e) { @@ -398,6 +611,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } else if (sendStaleBeforeNextReply) { sendStaleBeforeNextReply = false; sendMalformed(client, staleRequestId); + send(client, staleRequestId, 999); send(client, requestId, requests); } else if (malformedReply) { sendMalformed(client, requestId); diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md index 6d75d47d4..100ee6b22 100644 --- a/design/schema-aware-sender.md +++ b/design/schema-aware-sender.md @@ -729,8 +729,10 @@ After schema support is confirmed, required lookups use a fixed 30-second deadline. A cache-miss describe or forced refresh starts its own 30-second budget. This is not a public configuration surface. Timeout, unavailable metadata or request saturation throws a lookup error before accepting the -value. It cancels any partial row but does not permanently halt the sender. Do -not add automatic retries or separate timeout settings for lookup paths. +value. It cancels any partial row but does not permanently halt the sender. +Recoverable transport loss resends the pending lookup after reconnection within +its original deadline. Do not retry completed lookup errors or add separate +timeout settings for lookup paths. ### Minimal client integration @@ -1637,9 +1639,15 @@ bound for schemas, converters and pending buffers. Use one outstanding lookup because the producer owns row construction; no general request queue is needed. Its deadline starts at the producer call and includes waiting for the I/O thread, connection and response. Never reuse a -request ID; correlate completion with the connection that sent it. Timeout, -shutdown and connection loss release waiters. Ignore cancelled or unmatched -replies entirely, including their metadata. +request ID; correlate completion with the connection that sent it. Recoverable +transport loss retains the pending lookup, including one not yet sent. Resend +on the replacement connection with a fresh request ID within the original +deadline; repeated disconnects do not restart that deadline. The lookup can +time out while the store-and-forward loop continues reconnecting indefinitely. +Timeout, interruption, shutdown and terminal failure release waiters. Ignore +cancelled or unmatched replies entirely, including their metadata. Explicit +denial, unavailability and unsupported responses complete the lookup without +retrying it. Validate the fixed control header and request ID before parsing a direct reply's schema. An identifiable stale reply is ignored even if its schema body @@ -1647,9 +1655,11 @@ is malformed. Invalid headers that cannot safely identify a reply still fail the connection. Parse outside the coordinator monitor; if parsing fails, recheck the existing request identity and deadline before accepting the error as a current connection failure. Successful parsing retains the final check -before publication. Existing timeout and connection-loss paths remain -responsible for cancellation; response validation must not pre-complete a -request before recycling its connection. +before publication. A malformed current reply cancels its matching lookup +atomically with that identity check before recycling the connection; malformed +headers that cannot identify a reply cancel the connection's pending lookup. +Protocol errors retain this immediate failure policy even when transport loss +would preserve the lookup for reconnection. Cache known schemas and confirmed missing-table results, not denial or unavailability. A forced refresh removes the previous entry before lookup; @@ -1781,8 +1791,9 @@ They do not change the compatibility contract. of copying the server's one-byte truncation bug. 14. **Keep the schema deadline fixed.** Initial negotiation and its first describe share one 30-second end-to-end budget; later cache-miss describes - and forced refreshes each get their own 30-second budget. Do not add public - configuration, retries, another socket, a request queue or separate timeout + and forced refreshes each get their own 30-second budget. Transport retries + retain that original budget. Do not add public configuration, retries of + completed lookup errors, another socket, a request queue or separate timeout policy per path. 15. **Prefer narrow family helpers to a conversion framework.** A shared helper for BYTE/SHORT/INT numeric targets is justified because those setters have From 5c78a7d888819005d5879f268a24ed5b7be43a7e Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 18 Sep 2026 15:56:56 +0200 Subject: [PATCH 29/51] Prepare QWP row schemas in table() --- .../main/java/io/questdb/client/Sender.java | 6 +- .../qwp/client/QwpWebSocketSender.java | 168 ++++++++------ .../io/questdb/client/impl/PooledSender.java | 7 +- .../QwpSchemaSenderIntegrationTest.java | 213 +++++++++++++++--- .../qwp/client/QwpWebSocketSenderTest.java | 74 +++++- .../client/test/impl/SenderPoolTest.java | 49 ++++ 6 files changed, 396 insertions(+), 121 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index a9ec63bbf..a736052d4 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -758,13 +758,15 @@ default Sender shortColumn(CharSequence name, short value) { Sender symbol(CharSequence name, CharSequence value); /** - * Select the table for a new row. This is always the first method to start an error. It's an error to call other - * methods without calling this method first. + * Select the table and start a new row. Call this before every row, including consecutive rows for the same table. + * It's an error to add a column or complete a row without calling this method first. *
      * After calling this method you can start adding columns to the row and then call {@link #atNow()} or {@link #at(Instant)} * to finalize the row. You can then start a new row by calling this method again. *
      * If you want to cancel the current row, you can call {@link #cancelRow()}. + * QWP WebSocket senders also resolve and pin the row's schema contract here, so this call may wait for or fail + * schema discovery. Column setters do not initiate schema discovery. * * @param table name of the table * @return this instance for method chaining diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4679bef4d..bf332d6df 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -252,6 +252,12 @@ public class QwpWebSocketSender implements Sender { private int currentBatchMaxSymbolId = -1; private volatile int currentEndpointIdx = -1; private QwpTableBuffer currentTableBuffer; + // Authoritative row lifecycle state. A row is open immediately after a + // successful table(), even before any value has been appended. + private boolean currentRowOpen; + // Contract selected by table() for the open row. Null means an explicitly + // prepared legacy row; it never permits lazy negotiation in a setter. + private QwpSchemaBinding currentRowBinding; // Tracks currentTableBuffer.getBufferedBytes() at the last point pendingBytes // was made consistent (end of sendRow(), or right after a table switch). // sendRow() advances pendingBytes by (now - snapshot) and re-snaps, which @@ -1028,7 +1034,7 @@ public void at(long timestamp, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.designatedTimestamp(timestamp, unit); sendRow(); @@ -1050,7 +1056,7 @@ public void at(Instant timestamp) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.designatedTimestamp(timestamp); sendRow(); @@ -1068,7 +1074,6 @@ public void atNow() { checkNotClosed(); checkTableSelected(); try { - bindingForEffectiveWrite(); // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { @@ -1139,7 +1144,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.binaryColumn(columnName, value); return this; @@ -1169,7 +1174,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.binaryColumn(columnName, slice); return this; @@ -1194,7 +1199,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.binaryColumn(columnName, ptr, len); return this; @@ -1214,7 +1219,7 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.boolColumn(columnName, value); return this; @@ -1245,7 +1250,7 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.byteColumn(columnName, value); return this; @@ -1263,10 +1268,7 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { @Override public void cancelRow() { checkNotClosed(); - if (currentTableBuffer != null) { - currentTableBuffer.cancelCurrentRow(); - currentTableBuffer.rollbackUncommittedColumns(); - } + rollbackRow(); } /** @@ -1282,7 +1284,7 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.charColumn(columnName, value); return this; @@ -1392,6 +1394,10 @@ private void close0(boolean[] restoreInterrupt) { ? cursorSendLoop.getSynchronouslySurfacedError() : null; try { + // Closing never turns an unfinished row into an implicit write. + // Keep cancellation inside the protected cleanup path so even an + // unexpected rollback failure cannot skip transport teardown. + rollbackRow(); // Only drain when both the engine and the I/O loop are wired // up — close() is also called from createForTesting() teardown // and from connect() rollback paths where one or both may be null. @@ -1619,10 +1625,10 @@ public void setSlotLockReleasedForTesting(boolean isReleased) { @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { checkNotClosed(); - if (value == null || value.isNull()) return this; checkTableSelected(); + if (value == null || value.isNull()) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.decimalColumn(name, value); return this; @@ -1640,10 +1646,10 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { @Override public Sender decimalColumn(CharSequence name, Decimal128 value) { checkNotClosed(); - if (value == null || value.isNull()) return this; checkTableSelected(); + if (value == null || value.isNull()) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.decimalColumn(name, value); return this; @@ -1661,10 +1667,10 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { @Override public Sender decimalColumn(CharSequence name, Decimal256 value) { checkNotClosed(); - if (value == null || value.isNull()) return this; checkTableSelected(); + if (value == null || value.isNull()) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.decimalColumn(name, value); return this; @@ -1682,10 +1688,10 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { @Override public Sender decimalColumn(CharSequence name, CharSequence value) { checkNotClosed(); - if (value == null || value.length() == 0) return this; checkTableSelected(); + if (value == null || value.length() == 0) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.decimalColumn(name, value, currentDecimal256); return this; @@ -1704,10 +1710,10 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { @Override public Sender doubleArray(@NotNull CharSequence name, double[] values) { checkNotClosed(); - if (values == null) return this; checkTableSelected(); + if (values == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.doubleArray(name, values); return this; @@ -1725,10 +1731,10 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { @Override public Sender doubleArray(@NotNull CharSequence name, double[][] values) { checkNotClosed(); - if (values == null) return this; checkTableSelected(); + if (values == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.doubleArray(name, values); return this; @@ -1746,10 +1752,10 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { @Override public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { checkNotClosed(); - if (values == null) return this; checkTableSelected(); + if (values == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.doubleArray(name, values); return this; @@ -1767,10 +1773,10 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { @Override public Sender doubleArray(CharSequence name, DoubleArray array) { checkNotClosed(); - if (array == null) return this; checkTableSelected(); + if (array == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.doubleArray(name, array); return this; @@ -1790,7 +1796,7 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.doubleColumn(columnName, value); return this; @@ -1816,7 +1822,7 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.floatColumn(columnName, value); return this; @@ -1962,7 +1968,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.geoHashColumn(columnName, bits, precisionBits); return this; @@ -2000,7 +2006,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.geoHashColumn(columnName, value); return this; @@ -2388,7 +2394,7 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.intColumn(columnName, value); return this; @@ -2421,7 +2427,7 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.ipv4Column(columnName, address); return this; @@ -2466,12 +2472,12 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { @Override public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence address) { checkNotClosed(); + checkTableSelected(); if (address == null) { return this; } - checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.ipv4Column(columnName, address); return this; @@ -2515,7 +2521,7 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.long256Column(columnName, l0, l1, l2, l3); return this; @@ -2533,10 +2539,10 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l @Override public Sender longArray(@NotNull CharSequence name, long[] values) { checkNotClosed(); - if (values == null) return this; checkTableSelected(); + if (values == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2555,10 +2561,10 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { @Override public Sender longArray(@NotNull CharSequence name, long[][] values) { checkNotClosed(); - if (values == null) return this; checkTableSelected(); + if (values == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2576,10 +2582,10 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { @Override public Sender longArray(@NotNull CharSequence name, long[][][] values) { checkNotClosed(); - if (values == null) return this; checkTableSelected(); + if (values == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2597,10 +2603,10 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { @Override public Sender longArray(@NotNull CharSequence name, LongArray array) { checkNotClosed(); - if (array == null) return this; checkTableSelected(); + if (array == null) return this; try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2620,7 +2626,7 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.longColumn(columnName, value); return this; @@ -2703,6 +2709,7 @@ public void setCloseStartedHook(Runnable hook) { @Override public void reset() { checkNotClosed(); + rollbackRow(); // Reset ALL table buffers, not just the current one ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { @@ -2728,6 +2735,7 @@ public void reset() { currentTableBuffer = null; currentTableBufferSnapshotBytes = 0; currentTableName = null; + currentRowBinding = null; cachedTimestampColumn = null; cachedTimestampNanosColumn = null; } @@ -2933,7 +2941,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.shortColumn(columnName, value); return this; @@ -3053,7 +3061,7 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.stringColumn(columnName, value); return this; @@ -3073,7 +3081,7 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.symbol(columnName, value); return this; @@ -3091,17 +3099,12 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { @Override public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); - // Fast path: if table name matches current, skip hashmap lookup - if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { - return this; - } - // Prevent switching tables while a row is in progress - if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { - throw new LineSenderException("cannot switch tables while row is in progress" + if (currentRowOpen) { + throw new LineSenderException("cannot start a new row while a row is in progress" + " [currentTable=").put(currentTableName).put(']'); } - // Table changed - invalidate cached column references validateTableName(tableName); + ensureConnected(); cachedTimestampColumn = null; cachedTimestampNanosColumn = null; currentTableBuffer = tableBuffers.get(tableName); @@ -3117,7 +3120,18 @@ public QwpWebSocketSender table(CharSequence tableName) { // its last-snapped value (the in-progress-row guard above ensures // no column setters ran on it since the last consistency point). currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); - // Both modes accumulate rows until flush + try { + currentRowBinding = prepareRowBinding(); + currentRowOpen = true; + } catch (RuntimeException | Error e) { + currentRowBinding = null; + currentTableBuffer = null; + currentTableName = null; + currentTableBufferSnapshotBytes = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + throw e; + } return this; } @@ -3126,7 +3140,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.timestampColumn(columnName, value, unit); return this; @@ -3154,7 +3168,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.timestampColumn(columnName, value); return this; @@ -3182,7 +3196,7 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = bindingForEffectiveWrite(); + QwpSchemaBinding binding = currentRowBinding; if (binding != null) { binding.uuidColumn(columnName, lo, hi); return this; @@ -3878,7 +3892,7 @@ private void checkNotClosed() { } private void checkTableSelected() { - if (currentTableBuffer == null) { + if (!currentRowOpen) { throw new LineSenderException("table() must be called before adding columns"); } } @@ -4359,8 +4373,7 @@ private void ensureConnected() { } /** - * Selects the wire contract before the first effective value of a row. - * Empty table selection remains local. In asynchronous initial-connect mode, + * Selects the wire contract while table() prepares a row. In asynchronous initial-connect mode, * fresh senders use the legacy contract until schema support is confirmed; * recovered schema-framed data and senders that have already confirmed support * retain the strict schema path. @@ -4370,12 +4383,8 @@ private void ensureConnected() { * the cache; the first row after a flush or reset adopts whatever the cache holds * by then, which ACK schema feedback and reconnects keep current. */ - private QwpSchemaBinding bindingForEffectiveWrite() { + private QwpSchemaBinding prepareRowBinding() { QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); - if (currentTableBuffer.hasInProgressRow()) { - return pinned; - } - ensureConnected(); if (pinned != null && currentTableBuffer.getRowCount() > 0) { return pinned; } @@ -4437,7 +4446,7 @@ private static long remainingSchemaMillis(long deadlineNanos) { } private void ensureNoInProgressRow() { - if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { + if (currentRowOpen) { throw new LineSenderException( "Cannot flush while row is in progress. " + "Use sender.at(), sender.atNow(), or sender.cancelRow() first." @@ -5418,12 +5427,18 @@ private int symbolDeltaBaseline() { } private void rollbackRow() { - if (currentTableBuffer != null) { + if (currentRowOpen) { currentTableBuffer.cancelCurrentRow(); currentTableBuffer.rollbackUncommittedColumns(); + endRow(); } } + private void endRow() { + currentRowOpen = false; + currentRowBinding = null; + } + /** * Seals the current buffer and swaps to the other buffer. * Enqueues the sealed buffer for async sending. @@ -5497,8 +5512,6 @@ private void sealAndSwapBuffer() { * Rows buffer until flush (explicit or auto-flush). */ private void sendRow() { - ensureConnected(); - // Hard guard: a single row whose bytes exceed the server's wire cap // would flush as an oversize WS frame the server closes with // ws-close[1009]. nextRow() measures the row -- padding included, @@ -5518,7 +5531,7 @@ private void sendRow() { if (pendingRowCount == 0) { firstPendingRowTimeNanos = System.nanoTime(); // The batch's first row selects its wire contract; see - // bindingForEffectiveWrite. + // prepareRowBinding. isPendingBatchLegacy = currentTableBuffer.getSchemaBinding() == null; } pendingRowCount++; @@ -5532,6 +5545,11 @@ private void sendRow() { pendingBytes += bufferedNow - currentTableBufferSnapshotBytes; currentTableBufferSnapshotBytes = bufferedNow; + // The row belongs to the completed batch from this point onward. End + // the open-row state before auto-flush so a publication failure cannot + // roll the committed row back or invite the caller to append it twice. + endRow(); + if (shouldAutoFlush()) { flushPendingRows(transactional); } diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index 7b4e5f802..f431cb186 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -25,6 +25,7 @@ package io.questdb.client.impl; import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.std.Decimal128; @@ -166,7 +167,11 @@ public void close() { // above. A stale duplicate throws here, and the finally routes it // to discardBroken, which drops it on the same stale-generation // check under the pool lock. - slot.live(generation).flush(); + Sender delegate = slot.live(generation); + if (delegate instanceof QwpWebSocketSender) { + delegate.cancelRow(); + } + delegate.flush(); flushed = true; } finally { if (flushed) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index f77291b2d..552507712 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -65,6 +65,7 @@ public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exc .decimalColumn("value", "not-a-decimal") .atNow(); + sender.table("events"); sender.uuidColumn("failed_b", 11, 12); try { sender.decimalColumn("value", "not-a-decimal"); @@ -73,6 +74,7 @@ public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exc Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.decimalColumn("value", "2.5").atNow(); sender.flush(); @@ -91,11 +93,11 @@ public void testDecimalTextOverloadFormatsTextAndPreservesNullNoOps() throws Exc LongTextSchemaHandler handler = new LongTextSchemaHandler(941 + targetType, 951, targetType); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").decimalColumn("value", "00123.4500").atNow(); - sender.decimalColumn("value", "1.2300m").atNow(); - sender.decimalColumn("value", "Infinity") + sender.table("events").decimalColumn("value", "1.2300m").atNow(); + sender.table("events").decimalColumn("value", "Infinity") .decimalColumn("value", "not-a-decimal") .atNow(); - sender.decimalColumn("value", (CharSequence) null) + sender.table("events").decimalColumn("value", (CharSequence) null) .decimalColumn("value", "") .atNow(); sender.flush(); @@ -120,6 +122,7 @@ public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() thr try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) -1).atNow(); + sender.table("events"); sender.byteColumn("value", (byte) 99); try { switch (i % 3) { @@ -138,9 +141,10 @@ public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() thr Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.shortColumn("value", (short) 2).atNow(); - sender.intColumn("value", Integer.MIN_VALUE).atNow(); - sender.intColumn("value", 3).atNow(); + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.table("events").intColumn("value", 3).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).smallIntegerNumericTable( @@ -193,6 +197,7 @@ public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exc try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) -1).atNow(); + sender.table("events"); sender.shortColumn("value", (short) 99); try { sender.intColumn("failed_b", 1); @@ -201,10 +206,11 @@ public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exc Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.shortColumn("value", (short) 2).atNow(); - sender.intColumn("value", Integer.MIN_VALUE).atNow(); - sender.intColumn("value", 3).atNow(); - sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.table("events").intColumn("value", 3).atNow(); + sender.table("events").longColumn("value", Long.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).integerTemporalTable( @@ -257,6 +263,7 @@ public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Ex try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) -128).atNow(); + sender.table("events"); sender.shortColumn("value", (short) 99); try { sender.intColumn("failed_b", 1); @@ -265,9 +272,10 @@ public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Ex Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.shortColumn("value", Short.MAX_VALUE).atNow(); - sender.intColumn("value", Integer.MIN_VALUE).atNow(); - sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.table("events").intColumn("value", Integer.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).smallIntegerTextTable( @@ -319,6 +327,7 @@ public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Excep try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) 42).atNow(); + sender.table("events"); sender.shortColumn("value", (short) 99); try { sender.intColumn("failed_b", 1); @@ -327,9 +336,10 @@ public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Excep Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.shortColumn("value", (short) -7).atNow(); - sender.intColumn("value", Integer.MIN_VALUE).atNow(); - sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.table("events").intColumn("value", Integer.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).smallIntegerDecimalTable( @@ -378,8 +388,9 @@ public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { LongTextSchemaHandler handler = new LongTextSchemaHandler(1121, 1131, ColumnType.IPv4); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); - sender.intColumn("value", 0).atNow(); + sender.table("events").intColumn("value", 0).atNow(); + sender.table("events"); sender.intColumn("value", 0x0a000001); try { sender.intColumn("failed_b", 1); @@ -388,8 +399,9 @@ public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.intColumn("value", 1).atNow(); - sender.intColumn("value", -1).atNow(); + sender.table("events").intColumn("value", -1).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).intToIpv4Table("events", 1121, 1131); @@ -458,6 +470,7 @@ public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversR .binaryColumn("value", new byte[]{1}) .atNow(); + sender.table("events"); sender.decimalColumn("value", Decimal256.MAX_VALUE); try { sender.decimalColumn("failed_b", new Decimal64(1, 0)); @@ -466,15 +479,16 @@ public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversR Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.decimalColumn("value", new Decimal64(-125, 3)).atNow(); sender.table("events").atNow(); - sender.decimalColumn("value", new Decimal256( + sender.table("events").decimalColumn("value", new Decimal256( 0x161bcca7119915b5L, 0x0764b4abe8652979L, 0x7775a5f171950fffL, 0xffffffffffffffffL, 76)).atNow(); - sender.decimalColumn("value", new Decimal64(0, 0)).atNow(); + sender.table("events").decimalColumn("value", new Decimal64(0, 0)).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).decimalTextLifecycleTable("events", 781, 791); @@ -492,8 +506,8 @@ public void testNativeDecimalTextMatchesServerScalarFormattingForRawPublicValues sender.table("events") .decimalColumn("value", new Decimal128(Long.MAX_VALUE, -1L)) .atNow(); - sender.decimalColumn("value", wrappedScale).atNow(); - sender.decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); + sender.table("events").decimalColumn("value", wrappedScale).atNow(); + sender.table("events").decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).decimalTextTable( @@ -515,13 +529,14 @@ public void testNativeDecimalRollbackAndTargetScaleWithoutTableReselection() thr LongTextSchemaHandler handler = new LongTextSchemaHandler(601, 611, target); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").decimalColumn("value", new Decimal64(150, 2)).atNow(); - sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + sender.table("events").uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); try { sender.decimalColumn("value", Decimal128.fromLong(100_000_000_000_000_000L, 0)); Assert.fail("expected target precision overflow"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.decimalColumn("value", new Decimal64(25, 2)).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).decimal64Table("events", 601, 611, 4, 15_000, 2_500); @@ -722,13 +737,14 @@ public void testStringToGeoHashNullWidthsRollbackAndRecovery() throws Exception 551, 561, ColumnType.getGeoHashTypeWithBits(8)); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").stringColumn("value", "zz").atNow(); - sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); try { sender.stringColumn("value", "a"); Assert.fail("expected invalid GeoHash text"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.stringColumn("value", "04").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).geoHashTable("events", 551, 561, 8, 0xff, 1); @@ -772,13 +788,14 @@ public void testStringToLong256RollbackAndRecoveryWithoutTableReselection() thro try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").stringColumn("value", null); sender.binaryColumn("value", new byte[]{1}).atNow(); - sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); try { sender.stringColumn("value", "0x0"); Assert.fail("expected odd-length LONG256 rejection"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.stringColumn("value", "0x0123456789abcdeffedcba9876543210").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).long256Table("events", 431, 441); @@ -822,13 +839,14 @@ public void testStringToCharRollbackAndRecoveryWithoutTableReselection() throws LongTextSchemaHandler handler = new LongTextSchemaHandler(391, 401, ColumnType.CHAR); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").stringColumn("value", "A").atNow(); - sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + sender.table("events").uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); try { sender.binaryColumn("value", new byte[]{1}); Assert.fail("expected BINARY to CHAR rejection"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.stringColumn("value", "\ud83d").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).charTable("events", 391, 401, 'A', '?'); @@ -871,13 +889,14 @@ public void testTimestampToTextTargetsRollbackAndRecoverWithoutTableReselection( LongTextSchemaHandler handler = new LongTextSchemaHandler(351 + target, 361, target); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.MICROS).atNow(); - sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + sender.table("events").uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); try { sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); Assert.fail("expected timestamp range error"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.timestampColumn("value", 1, ChronoUnit.MILLIS).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).timestampTextTable( @@ -933,6 +952,7 @@ public void testStringToTimestampTargetsUsePinnedWireAndRollback() throws Except Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.stringColumn("value", "1969-12-31T23:59:59.999999Z").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).longTimestampTable( @@ -963,6 +983,7 @@ public void testStringToDateUsesExactTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.stringColumn("value", "1969-12-31").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()) @@ -1018,6 +1039,7 @@ public void testDoubleArrayUsesExactTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()) @@ -1091,6 +1113,7 @@ public void testLongArrayRejectionRollsBackWithoutRefresh() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); sender.flush(); // Only the two completed DOUBLE[] rows reach the wire; failed_b is rolled back. @@ -1127,6 +1150,7 @@ public void testLongArrayRejectionKeepsPinnedSnapshotAfterSchemaChange() throws // A supported input of the new rank is still rejected against the // pinned snapshot; the batch does not discover the rank change. try { + sender.table("events"); sender.doubleArray("value", new double[]{3.0, 4.0}); Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { @@ -1213,6 +1237,7 @@ public void testLongToTimestampTargetsUseRawUnitsAndRollback() throws Exception } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.longColumn("value", Long.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()) @@ -1302,6 +1327,7 @@ public void testFloatingToTextTargetsUsePinnedWireAndRollback() throws Exception Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(1)); else sender.doubleColumn("value", Double.longBitsToDouble(1)); sender.atNow(); @@ -1372,6 +1398,7 @@ public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L).atNow(); sender.flush(); @@ -1411,6 +1438,7 @@ public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.uuidColumn("failed_b", 1, 2); try { sender.ipv4Column("value", "not-an-ip"); @@ -1419,6 +1447,7 @@ public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.ipv4Column("value", ".....255.1.2.3......").atNow(); sender.flush(); @@ -1483,6 +1512,7 @@ public void testGeoHashTargetsUsePinnedTargetWireAndRollback() throws Exception Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.geoHashColumn("value", 0x12345, 20).atNow(); sender.flush(); @@ -1549,6 +1579,7 @@ public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.long256Column("value", 1, 2, 3, 4).atNow(); sender.flush(); @@ -1613,7 +1644,8 @@ public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - // The failed middle row is auto-cancelled; continue without selecting the table again. + // The failed middle row is auto-cancelled; start a fresh row. + sender.table("events"); sender.longColumn("value", Long.MAX_VALUE).atNow(); sender.flush(); @@ -1673,6 +1705,7 @@ public void testAsymmetricUuidRowsArePinnedAndInvalidMiddleRowRollsBack() throws } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } + sender.table("events"); sender.uuidColumn("id", 0x2122232425262728L, 0x3132333435363738L).atNow(); sender.flush(); byte[] frame = handler.awaitDataFrame(); @@ -1753,6 +1786,7 @@ public void testKnownAdoptionWaitsForBatchBoundary() throws Exception { // The pending batch keeps its inferred snapshot: a conflicting input // fails against the inferred LONG and the failed row rolls back whole. try { + sender.table("events"); sender.stringColumn("failed_b", "partial"); sender.stringColumn("id", "11121314-1516-1718-0102-030405060708"); Assert.fail("expected inferred type conflict"); @@ -1780,11 +1814,13 @@ public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throw Sender sender = sender(server)) { sender.table("events").longColumn("id", 1).atNow(); try { + sender.table("events"); sender.stringColumn("id", "different inferred type"); Assert.fail("expected inferred type conflict"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.longColumn("id", 3).atNow(); sender.flush(); FrameReader reader = new FrameReader(handler.awaitDataFrame()); @@ -1822,12 +1858,14 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr // The pending batch never consults the server: the conflict is // local and only the partial row rolls back. try { + sender.table("events"); sender.stringColumn("failed_only", "partial"); sender.stringColumn("id", "different inferred type"); Assert.fail("expected inferred type conflict"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } + sender.table("events"); sender.longColumn("id", 3).atNow(); long fsn = sender.flushAndGetSequence(); Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); @@ -1841,10 +1879,9 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr // The server could not describe the table for the stale frame, so // its ACK invalidated the cache. The next batch looks the table up - // again and reports the lookup failure from the setter. - sender.table("events"); + // again and reports the lookup failure from table(). try { - sender.longColumn("id", 5); + sender.table("events"); Assert.fail("expected lookup failure"); } catch (LineSenderSchemaException e) { Assert.assertEquals(reasons[i], e.getReason()); @@ -1865,7 +1902,7 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr } @Test - public void testNullNoopDoesNotTriggerNegotiation() throws Exception { + public void testNullNoopUsesContractPreparedByTable() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); try (TestWebSocketServer server = schemaServer(handler); @@ -1876,7 +1913,7 @@ public void testNullNoopDoesNotTriggerNegotiation() throws Exception { sender.longArray("values", (long[][][]) null); sender.longArray("values", (LongArray) null); sender.ipv4Column("address", (CharSequence) null); - Assert.assertEquals(0, handler.describeRequests.get()); + Assert.assertEquals(1, handler.describeRequests.get()); sender.cancelRow(); } }); @@ -1998,6 +2035,27 @@ public void testOldPeerUsesLegacyGeoHashWireAndNeverDescribes() throws Exception }); } + @Test + public void testTablePinsSchemaBeforeFirstValue() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 51, 52); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events"); + Assert.assertEquals(1, handler.describeRequests.get()); + + // A schema change after table() must not alter the open row. + handler.version = 53; + sender.uuidColumn("id", 1, 2).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).uuidTable("events", 51, 52, 1, 2); + Assert.assertEquals("setters and completion must not discover schema", 1, + handler.describeRequests.get()); + } + }); + } + @Test public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { assertMemoryLeak(() -> { @@ -2016,6 +2074,7 @@ public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { } // The rejection neither looks up metadata nor splits the batch: the // next row joins the block pinned to version 52. + sender.table("events"); sender.uuidColumn("id", 3, 4).atNow(); long fsn = sender.flushAndGetSequence(); new FrameReader(handler.awaitDataFrame()).uuidTable("events", 51, 52, 1, 2, 3, 4); @@ -2145,6 +2204,86 @@ public void testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect() t }); } + @Test(timeout = 10_000) + public void testAsyncRowPreparedOfflineStaysLegacyWhenHandshakePrecedesFirstValue() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); + TestWebSocketServer server = null; + try { + Instant value = Instant.ofEpochSecond(3, 456_789_123); + sender.table("events"); + + SchemaHandler handler = new SchemaHandler( + QwpSchemaProtocol.RESULT_KNOWN, 73, 74, true, -1); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + + // The row contract was fixed by table(), before the connection existed. + sender.timestampColumn("ts", value).atNow(); + sender.flush(); + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyTimestampTable("events", "ts", 3_456_789L); + new FrameReader(frames.get(1)).schemaTimestampTable( + "events", 73, 74, "ts", 3_456_789_123L); + Assert.assertEquals(1, handler.describeRequests.get()); + } finally { + sender.close(); + if (server != null) { + server.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testCancellingAsyncRowPreparedOfflineDoesNotPinLegacyBatch() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); + TestWebSocketServer server = null; + try { + sender.table("events"); + + SchemaHandler handler = new SchemaHandler( + QwpSchemaProtocol.RESULT_KNOWN, 75, 76, true, -1); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + + sender.cancelRow(); + sender.table("events").timestampColumn( + "ts", Instant.ofEpochSecond(3, 456_789_123)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).schemaTimestampTable( + "events", 75, 76, "ts", 3_456_789_123L); + Assert.assertEquals(1, handler.describeRequests.get()); + } finally { + sender.close(); + if (server != null) { + server.close(); + } + } + }); + } + @Test public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exception { assertMemoryLeak(() -> { @@ -2196,15 +2335,22 @@ public void testSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() thr + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(1_000); - sender.table("events"); try { - sender.uuidColumn("id", 1, 2); + sender.table("events"); Assert.fail("expected schema negotiation deadline"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); } + try { + sender.uuidColumn("id", 1, 2); + Assert.fail("failed table() must leave no open row"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table() must be called")); + } + Assert.assertEquals("a setter after failed table() must not retry lookup", 1, + handler.describeRequests.get()); handler.respondToDescribe = true; - sender.uuidColumn("id", 3, 4).atNow(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); sender.flush(); Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); @@ -2278,9 +2424,8 @@ public void testAsyncRecoveredSchemaBacklogDoesNotFallBackToLegacy() throws Exce + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + "close_flush_timeout_millis=0;")) { ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(250); - sender.table("events"); try { - sender.uuidColumn("id", 3, 4); + sender.table("events"); Assert.fail("recovered schema-required backlog fell back to legacy encoding"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index cd598fa70..9deef6d3d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -286,6 +286,50 @@ public void testCancelRowNoOpWithoutTable() throws Exception { }); } + @Test + public void testEveryRowRequiresTableAndEmptyOpenRowRejectsLifecycleOperations() throws Exception { + assertMemoryLeak(() -> { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { + assertTableRequired(() -> sender.stringColumn("ignored", null)); + + sender.table("t"); + try { + sender.table("t"); + Assert.fail("expected second table() to reject the open row"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("row is in progress")); + } + try { + sender.flush(); + Assert.fail("expected flush() to reject the empty open row"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("row is in progress")); + } + try { + sender.flushAndGetSequence(); + Assert.fail("expected flushAndGetSequence() to reject the empty open row"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("row is in progress")); + } + try { + sender.drain(1); + Assert.fail("expected drain() to reject the empty open row"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("row is in progress")); + } + + sender.cancelRow(); + assertTableRequired(sender::atNow); + sender.table("t").atNow(); + assertTableRequired(sender::atNow); + sender.table("t").longColumn("v", 1).atNow(); + sender.cancelRow(); + assertTableRequired(() -> sender.longColumn("v", 2)); + } + }); + } + @Test public void testCloseIdemponent() throws Exception { assertMemoryLeak(() -> { @@ -337,7 +381,7 @@ public void testDoubleColumnAfterCloseThrows() throws Exception { } @Test - public void testFlushAppendFailureDoesNotLeaveMicrobatchBufferInUse() throws Exception { + public void testAutoFlushAppendFailureRetainsCommittedRowAndEndsOpenRow() throws Exception { assertMemoryLeak(() -> { try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { })) { @@ -352,17 +396,20 @@ public void testFlushAppendFailureDoesNotLeaveMicrobatchBufferInUse() throws Exc long minSegmentBytes = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; CursorSendEngine engine = new CursorSendEngine(null, minSegmentBytes, minSegmentBytes, 1L); try (QwpWebSocketSender sender = QwpWebSocketSender.connect( - "localhost", port, null, Integer.MAX_VALUE, 0, 0L, null, + "localhost", port, null, 1, 0, 0L, null, false, engine, 0L)) { - sender.table("t").longColumn("v", 1L).atNow(); - try { - sender.flushAndGetSequence(); + sender.table("t").longColumn("v", 1L).atNow(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { Assert.assertTrue(e.getMessage().contains("cursor SF append failed")); } + Assert.assertEquals("auto-flush failure must retain the committed row", 1, + sender.getTableBuffer("t").getRowCount()); + sender.table("t"); + sender.cancelRow(); + MicrobatchBuffer buffer0 = getMicrobatchBuffer(sender, "buffer0"); MicrobatchBuffer buffer1 = getMicrobatchBuffer(sender, "buffer1"); Assert.assertFalse( @@ -419,7 +466,8 @@ public void testIpv4ColumnStringNullReturnsThis() throws Exception { // way it round-trips as SQL NULL. The call must not throw and must // return the sender so chained builders keep working. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); Assert.assertSame(sender, sender.ipv4Column("addr", null)); } @@ -541,9 +589,8 @@ public void testNullArgsAfterCloseAllThrowSenderClosed() throws Exception { public void testNullArrayReturnsThis() throws Exception { assertMemoryLeak(() -> { try (QwpWebSocketSender sender = createUnconnectedSender()) { - // Null arrays should be no-ops and return sender - Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); - Assert.assertSame(sender, sender.longArray("x", (long[]) null)); + assertTableRequired(() -> sender.doubleArray("x", (double[]) null)); + assertTableRequired(() -> sender.longArray("x", (long[]) null)); } }); } @@ -849,6 +896,15 @@ private static MicrobatchBuffer getMicrobatchBuffer(QwpWebSocketSender sender, S return (MicrobatchBuffer) field.get(sender); } + private static void assertTableRequired(Runnable operation) { + try { + operation.run(); + Assert.fail("expected table() requirement"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table() must be called")); + } + } + private static QwpWebSocketSender connectLegacySender( TestWebSocketServer server, int autoFlushRows, diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java index 1aedb3ea1..18338e692 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java @@ -26,6 +26,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.test.tools.TestUtils; @@ -34,6 +36,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Proxy; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** @@ -53,6 +56,52 @@ public class SenderPoolTest { private static final String DEAD_HTTP_CONFIG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; + @Test + public void testQwpCloseCancelsEmptyAndPartialRowsBeforeReusingSlot() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { + })) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + String config = "ws::addr=localhost:" + server.getPort() + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"; + try (SenderPool pool = new SenderPool(config, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { + Sender first = pool.borrow(); + Object slot = slotOf(first); + Field delegateField = slot.getClass().getDeclaredField("delegate"); + delegateField.setAccessible(true); + QwpWebSocketSender delegate = (QwpWebSocketSender) delegateField.get(slot); + + first.table("events"); + first.close(); + Assert.assertEquals("table-only close must publish nothing", -1, + delegate.getCursorEngineForTesting().publishedFsn()); + + Sender second = pool.borrow(); + Assert.assertSame(slot, slotOf(second)); + Assert.assertThrows(LineSenderException.class, second::atNow); + second.table("events").longColumn("v", 1).atNow(); + second.table("events").longColumn("partial_only", 2); + second.close(); + Assert.assertEquals("close must publish the completed row", 0, + delegate.getCursorEngineForTesting().publishedFsn()); + Assert.assertEquals("the incomplete row must not retain its new column", 1, + delegate.getTableBuffer("events").getColumnCount()); + Assert.assertEquals(0, delegate.getTableBuffer("events").getRowCount()); + + try (Sender third = pool.borrow()) { + Assert.assertSame(slot, slotOf(third)); + Assert.assertThrows(LineSenderException.class, () -> third.longColumn("v", 3)); + third.table("events").longColumn("v", 3).atNow(); + } + Assert.assertEquals("the next borrower can publish its own row", 1, + delegate.getCursorEngineForTesting().publishedFsn()); + } + } + }); + } + @Test public void testBorrowReturnRecyclesSameDecorator() throws Exception { try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { From 86e5d8185d11851cfb84c157c0223c0f59552179 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 18 Sep 2026 15:58:18 +0200 Subject: [PATCH 30/51] remove temporary design doc --- design/schema-aware-sender.md | 2180 --------------------------------- 1 file changed, 2180 deletions(-) delete mode 100644 design/schema-aware-sender.md diff --git a/design/schema-aware-sender.md b/design/schema-aware-sender.md deleted file mode 100644 index 100ee6b22..000000000 --- a/design/schema-aware-sender.md +++ /dev/null @@ -1,2180 +0,0 @@ -# Schema-aware sender: schema-directed encoding - -Status: schema-directed encoding is implemented on the development branch; -design revision 71. Scope: QWP v1 over WebSocket, with an automatically -negotiated schema extension, legacy-server compatibility and companion server -changes. The committed implementation contains the protocol, Sender integration -and conversions through iteration 2.46 at client `59ade916` and server -`ea019a99f3`, whose submodule pins that exact client revision. A released-binary -compatibility gate is implemented and locally green. The intentional, -server-supported public-setter conversion catalogue is complete. Revision 69 -defines best-effort legacy encoding during a genuinely asynchronous cold start; -revision 70 reduces its integration to one existing sticky schema requirement. -Revision 71 restores CHAR-to-STRING/VARCHAR compatibility with target-native -UTF-8 encoding. That conversion is implemented and locally validated in the -working tree. The explicit server-contract boundaries below remain excluded -rather than emulated. - -## Goal and contract - -On supporting servers, convert values to the server column's type before -buffering them. Conversion failures are reported by the setter and cancel only -the partial row. Completed rows stay buffered. Against -old servers, retain the existing inferred-type behavior and server-side -conversions; those writes do not gain early schema-mismatch detection. - -For a server UUID column, `stringColumn("id", validUuid)` parses locally and -appends a binary UUID; `stringColumn("id", "invalid")` throws -`LineSenderException`. `uuidColumn("id", lo, hi)` writes directly to the same -UUID buffer. The setter describes the input type; the server schema determines -the buffered representation. - -This replaces revision 4's compatibility checks. Knowing that STRING-to-UUID -is supported cannot validate a particular string. The client now performs the -conversion, rather than merely checking an acceptance mask. - -Schema mode has three parts: - -1. Obtain the server schema synchronously on first use or a cache miss. -2. Bind columns to that schema and convert every effective value before append. -3. Pin each table block to its encoding snapshot. ACKs/NACKs provide newer - schemas for subsequent blocks, without changing already converted data. - -The guarantee is local conversion validity against the pinned schema, not -eventual server acceptance. Concurrent DDL, auto-creation races and other -server-side failures can still reject a batch. - -## Implementation status - -Current working tree: iteration 2.47 implements CHAR-to-STRING/VARCHAR. It -preserves native CHAR storage, encodes non-surrogate characters directly as -target-native VARCHAR wire data for either text target, and rejects a lone -UTF-16 surrogate locally. All 3,721 client tests pass. The complete 138-test -server E2E class removes both CHAR regressions and passes the dedicated Unicode -case; its remaining 2 failures and 20 errors are the previously catalogued -DATE/GEOHASH and stale legacy-expectation cases. D069 records the decision and -evidence. - -Current checkpoint: iteration 2.46 is committed as client `59ade916` and pinned -by server `ea019a99f3`. It implements revision 70's asynchronous cold-start -behavior. Before schema support is known, an `ASYNC` sender uses the legacy -contract; after a supporting handshake, the existing engine requirement makes -schema mode sticky. Recovered schema-framed backlog bypasses the legacy branch. -The production change is one branch in `bindingForEffectiveWrite()` plus API -documentation; it adds no state, protocol field, persisted format or send-time -conversion. - -Iteration 2.45 at client `bc16c164` and server `02af9a6f31` added all public -DOUBLE-array representations to canonical DOUBLE-array targets of exactly the -same rank. The intentional conversion inventory is complete. This does not turn -the separately documented unsafe or undefined server paths into supported -conversions. - -The standing compatibility gate now runs three real process combinations: -current client against QuestDB 10.0.1, client 1.3.9 against the current server, -and current client against the current server. It proves legacy versus schema -encoding from stored values, uses checksum-pinned released artifacts, and runs -in both the stable-client CI workflow and the release pipeline. Exact no-send, -watermark and replay behavior after a confirmed sender reaches an old endpoint -remains covered by deterministic socket and replay tests; it does not need a -second multi-process failover harness. D056 records the gate and local evidence. - -Schema discovery, pinned identity framing, safe replay and ACK/NACK feedback -are implemented and tested. Local UUID and LONG-to-numeric conversion now uses -a non-owning binding over a caller-owned `QwpTableBuffer`; the caller retains -row completion, rollback and buffer lifetime. - -First-use lookup and forced refresh now run through the existing cursor I/O -loop and share a bounded latest-schema cache with ACK/NACK feedback. Tests -cover lookup failure/lifecycle handling and a real-server lookup, schema change, -ACK-driven cache update and same-buffer rebind, including fragmented responses. - -Primitive timestamp units from NANOS through DAYS and `Instant` conversion into -non-designated timestamp columns are implemented and tested against real-server -ingestion. Coverage includes range boundaries, target precision, nulls, rollback, -compressed encoding, recovered delivery and deliberate legacy differences. -STRING-to-timestamp parsing is accepted in iteration 2.27. Designated timestamp row completion is -included in the Sender integration described below. - -Text and symbol inputs into STRING, VARCHAR and SYMBOL targets are implemented -and tested with 90 shared cases, exact wire/stored-value checks, legacy value -parity and local-dictionary SF recovery. Client tests also cover Sender-owned -global symbol encoding. - -Boolean inputs and strict text-to-boolean parsing are implemented and tested -against real-server acceptance/rejection, target-native omissions, packed nulls -and recovered delivery. - -FLOAT/DOUBLE input conversion into the six numeric targets is implemented and -tested: NaN means missing and out-of-range integer conversions fail locally. -Coverage includes raw-bit rounding boundaries, legacy differences, rollback -and recovered delivery. D025/D026 record the decisions and verification. - -BYTE/SHORT/INT input conversion into the six numeric targets is implemented in -iteration 2.37. One narrow helper performs target selection, range checks and -direct target-width append; it does not change the established LONG path. -Shared component, public-Sender, recovered-frame and real-server tests cover 64 -boundary vectors, exact wire/SQL values, rollback, refresh, inference and the -stable INT null contract. D055 records the local acceptance evidence. - -STRING input conversion into the six numeric targets is implemented and tested -with 429 shared cases, exact wire/stored values, legacy acceptance/rejection, -null/sentinel contexts, rollback and recovered delivery. Parsing preserves -target-specific syntax, including floating suffixes; overflowing LONG text -fails locally under the approved D027 compatibility exception. - -BINARY targets now accept all three binary input forms and STRING input encoded -as UTF-8, using native BINARY wire representation before persistence. Component and -real-server tests cover exact bytes, source ownership, null versus empty, -duplicate suppression, local errors and rollback. A shared variable-width -append repair also preserves completed rows if string encoding throws midway; -successful legacy writes retain their encoding. D029 records the accepted slice, -independent reviews and complete verification evidence. - -The textual decimal overload now behaves as a DECIMAL256 input in schema mode. -Known decimal targets use their declared precision and scale; STRING/VARCHAR -targets receive canonical fixed-point text. Parsed `NaN` and infinities write -target NULL, while Java `null` and empty input remain no-ops. A missing column -infers DECIMAL256 at the first finite value's natural scale; a null-only block -uses wire scale zero without locking later batches to that scale. The Sender -uses its existing rollback and one-refresh lifecycle, with no new retained -converter state or per-value allocation. - -Both IPv4 overloads now use the server schema in schema mode. An IPv4 target -keeps the four-byte value; STRING and VARCHAR receive canonical dotted-quad -text through the existing reusable numeric sink. Packed zero is written as a -bitmap NULL for every target. A Java null text reference remains a no-op. Exact -`"null"` (case-insensitive) and `"0.0.0.0"` strings remain rejected, while -legacy dotted aliases such as `".0.0.0.0."` remain accepted and normalize to -NULL. Malformed input fails locally with `INVALID_VALUE`, including strings -made only of dots. - -`long256Column` now uses the server schema in schema mode. A LONG256 target -keeps the four limbs in native 32-byte form; STRING and VARCHAR receive the -server's canonical lowercase `0x` text through the reusable numeric sink. Four -`Long.MIN_VALUE` limbs become an explicit bitmap NULL for every supported -target, while any partial match remains a value. Missing columns still infer -LONG256. Legacy mode keeps its existing native wire data and server-side -conversion behavior. Shared client/server vectors, exact wire checks, public -Sender ingestion, rollback, schema rebind, legacy framing and persisted-frame -replay cover the slice. D059 records the decision and evidence. - -Both `geoHashColumn` overloads now use the server schema in schema mode. A -GEOHASH target keeps the masked packed value only when its precision exactly -matches the source. STRING and VARCHAR receive the server's fixed-width binary -text, one character per source bit, through the existing reusable sink. Bits -above the declared precision remain ignored. Omission is the only GEOHASH null -operation; all packed values are valid, including byte-aligned all-one values. -Missing columns still infer the source precision, while legacy mode keeps its -original native frame. Shared vectors, exact wire checks, public-Sender -ingestion, local-error rollback, schema rebind, old-peer framing and persisted -frame replay cover the slice. D060 records the decision and evidence. - -BYTE, SHORT and INT inputs now select DATE, TIMESTAMP and TIMESTAMP_NS target -wire types directly; LONG also selects DATE. Values remain raw counts in the -target's unit. There is no implicit unit conversion: DATE means epoch -milliseconds, TIMESTAMP means epoch microseconds and TIMESTAMP_NS means epoch -nanoseconds. INT and LONG source sentinels become explicit bitmap NULLs; BYTE -and SHORT have no source-null sentinel. One existing small-integer helper and -the established LONG switch perform the append, with no new converter, retained -state, allocation or server production code. Shared exact-wire and real-server -vectors, partial-row rollback, parameter rejection and a DATE-to-TIMESTAMP_NS -schema rebind cover the slice. D061 records the decision and evidence. - -BYTE, SHORT and INT inputs now also select STRING, VARCHAR and SYMBOL target -wire types directly. The existing reusable integer text sink formats the widened -value once; STRING and VARCHAR use the same VARCHAR wire representation, while -SYMBOL uses the existing global dictionary path. `Integer.MIN_VALUE` remains a -source NULL and is written through the target bitmap. BYTE and SHORT have no -source-null sentinel. No source-width formatter, converter object, retained -value, per-row String or server production code was added. A shared 30-case -corpus, exact-wire tests, generation rebind, legacy isolation, public-Sender -real-server ingestion and A/error/C rollback cover the slice. D062 records the -decision and evidence. - -BYTE, SHORT and INT inputs now select DECIMAL8, DECIMAL16, DECIMAL32, -DECIMAL64, DECIMAL128 and DECIMAL256 target representations directly. All three -sources have scale zero, then reuse the established exact integer-decimal tail: -the value is rescaled to the target scale, checked against the declared -precision and appended in the target storage width. `Integer.MIN_VALUE` remains -a source NULL and records the target scale with a bitmap NULL; BYTE and SHORT -have no source-null sentinel. One shared helper also replaces the identical -LONG decimal tail without changing its behavior. Shared exact-wire and -real-server vectors, precision rejection, partial-row rollback and a decimal -precision/scale schema rebind cover the slice. D063 records the decision and -evidence. - -INT input now selects target-native IPv4 wire representation. The input's -`Integer.MIN_VALUE` sentinel becomes an explicit bitmap NULL; the integer value -zero also becomes NULL because zero is the target IPv4 sentinel. Every other -32-bit pattern is appended unchanged. BYTE, SHORT and LONG remain unsupported -for IPv4, so this is not a general integer/address conversion. Exact-wire tests, -public-Sender ingestion, partial-row rollback, parameter rejection and a -DECIMAL-to-IPv4 schema rebind cover the slice. D064 records the decision and -evidence. - -STRING input now selects target-native DATE wire representation after local -parsing. It preserves the server's current five-step grammar order: PostgreSQL -date-time, date-only, date with zone, date-time with milliseconds and zone, -UTC date-time with exactly three fractional digits, then raw epoch -milliseconds. `Long.MIN_VALUE`, whether parsed as raw text or produced by -calendar arithmetic, becomes a bitmap NULL. The parser reuses the existing -timestamp calendar and timezone tables with millisecond arithmetic; it does -not add a general format compiler or allocate a date object per value. A narrow -guard rejects an extreme negative-year/named-zone case where the current server -throws an unchecked exception instead of returning a conversion error. A -53-case shared corpus, a 100,015-input differential over the server's normal -return/error domain, exact wire checks, public-Sender ingestion, partial-row -rollback and a DATE-to-VARCHAR generation rebind cover the slice. D065 records -the decision and evidence. - -All public DOUBLE-array inputs now select target-native DOUBLE_ARRAY storage -when the server schema carries a canonical strong DOUBLE array of the same rank. -The primitive overloads supply ranks 1, 2 and 3; `DoubleArray` exposes its live -rank for dimensions 1 through 32. Known-rank mismatches fail before append, and -a confirmed missing column pins the first effective rank in its existing buffer -definition so later rows cannot form a mixed-rank block. Null references remain -no-ops, empty shapes remain values, and duplicate first-value-wins behavior is -unchanged. The implementation reuses the existing shape/data buffer and encoder; -it adds no element converter, copy, array parameter format or server production -code. Exact-wire, public socket, real-server shape/rank and partial-row recovery -tests cover the slice. D066 records the decision and evidence. - -Iteration 2.15 is accepted as a bounded, unreleased Sender integration. -Public setters select the negotiated mode, use server-directed conversions, -cancel failed partial rows and retain completed rows under their original -schema. Both designated timestamp units and cold `atNow()` are integrated. -Eight real-server tests cover these operations, relevant/unrelated DDL, -schema lookups, auto-flush, reset and close. Fifteen public-Sender socket -tests separately cover exact wire encoding, legacy fallback, one-way upgrade, -startup timeout/interruption, pinned generations and split frames. -At the 2.22 checkpoint, the broader server regression selection still had 38 -classified failures; -this is not a release-ready client. Compatibility with a non-confirming socket -peer is tested; an actual older QuestDB distribution was not run in this slice. -The fixed 30-second schema deadline is settled policy and is not part of the -remaining conversion plan. Complete conversion -coverage gates release, not the first integrated test. Until that -coverage is complete, this is an unreleased development increment: unsupported -schema-mode inputs fail explicitly rather than falling back to server conversion. -The [conversion backlog](#conversion-backlog-server-source-audit) inventories -every public input family against the current server, including missing identity -setters, unsupported pairs and unresolved compatibility boundaries. -The [implementation journal](schema-aware-sender-journal.md) -records decisions, iteration scope and verification evidence. - -Iteration 2.16 is accepted: two real-server tests create file-backed backlog -through public Sender setters, withhold data from the server and forcibly stop -the producer JVM. A fresh Sender replays byte-identical QWP payloads and stores -exact A/C rows after rejecting partial B. Coverage includes UUID/SYMBOL values, -two pinned UUID-to-VARCHAR schema generations, and reopening after ACK without -replaying acknowledged rows. The 230-test server gate and three seeded repeats -passed against the unchanged 2.15 client artifact. This increment changes only -tests and documentation; it proves neither power-loss durability nor a stronger -delivery guarantee, and does not close the release gaps above. D031 records -the evidence and review findings. - -Iteration 2.17's first repair group is accepted: legacy conversion comparisons -now send unflagged source-typed QWP frames instead of accidentally using schema -conversion on both sides. Original expected values are preserved, with explicit -wire assertions for source types, values and nulls. All 25 tests in the four -affected suites, the 251-test regression selection and three seeded repeats -passed. The original 333-test selection remains red with 56 failures/errors, -down from 58 known contract/fixture failures. Two additional disk-space failures -were resolved through the test JVM's startup configuration, not changed -expectations. No production changes or Sender opt-out were introduced. D032 -records the evidence and the next functional priority: missing-table/column -inference through the existing binding and buffer ownership. - -Iteration 2.18 is accepted for confirmed-missing inference in already-supported -setter families. Missing tables and columns use native inferred types in the -existing buffer; server-provided types still govern known columns. No separate -schema registry or new stored state was added. Same-row duplicates are ignored -before type checking; later-row inferred type conflicts use the existing -schema-refresh path. New metadata never relabels completed rows. Server creation -policy and authorization remain authoritative. -The client gate passed 414 tests plus two packaging checks; the server gate -passed 261 tests, with three seeded repeats of 16 integration/recovery tests. -The original 333-test selection has 54 failures/errors, down from 56, with no -new failing methods. D033 records the implementation, review corrections and -remaining diagnostic/conversion work. This does not complete unsupported input -families or make the client release-ready. - -Iteration 2.19 is accepted as a test-only diagnostic repair. Seven tests now -assert typed local errors and observable recovery. Independent legacy frames -still verify server rejection of BINARY-to-text/SYMBOL. The mismatch test waits -for ACK feedback before asserting the conversion error. No production behavior -or client artifact changed. The 268-test regression gate and three repeats of -23 integration/recovery cases passed. The unchanged 333-test selection has -47 failures/errors: exactly the seven repaired methods disappeared, with no -new failing methods. The remaining set is 38 missing-conversion cases, five -approved schema/legacy expectation differences and four obsolete fixtures. -D034 records evidence and a pending source-null decision for future text -conversions; this iteration does not approve another compatibility exception. - -Iteration 2.20 is accepted for source-null characterization, not conversion -implementation. Three real-server tests prove bitmap-dependent LONG/UUID text -results and current public Sender rejection/recovery across all five pairs. -The bitmap-present LONG result is non-null text `"null"`, correcting the earlier -decimal-minimum prediction. The 271-test regression gate and three repeats of -26 cases passed; the original 333-test selection retains the exact same 47 -failing methods. Production code and the client artifact are unchanged. -D035 records the evidence. The [text-null rules](#source-null-rules-for-text-output) -were proposals at that checkpoint; D039 subsequently approves LONG only. - -The [remaining-failures plan](schema-aware-sender-remaining-failures-plan.md) -assigns all 47 failing methods to small, independently tested increments: -nine expectation/fixture repairs, then 38 conversion cases. It preserves -independent legacy tests and requires public Sender E2E coverage in every -increment. D036 records the source audit and sequencing; this is a plan, -not another accepted implementation iteration or the full release checklist. - -Iteration 2.21 accepts phase 1a of that plan. Three public Sender Boolean tests -now assert target-native omissions; two floating-point legacy diagnostics send -independent unflagged frames. A new public Sender test preserves NaN and -positive-2^63 boundary/recovery coverage. All 25 focused tests, the expanded -277-test green gate and three repeats of 32 cases passed. The original 333-test -selection has 42 failing methods: exactly the five repaired cases disappeared, -with no new failure. No production code or client artifact changed. D037 records -the evidence. Four direct-buffer fixtures and all 38 missing-conversion cases -remained after that increment. - -Iteration 2.22 accepts phase 1b: the four direct-buffer fixtures now send -standalone unflagged legacy frames, preserving their original stored values. -They assert GeoHash precision/bitmap/packed bytes, literal named LONG designated -timestamps in both units, and legacy ACK framing. Public `at()` coverage remains -separate. All 37 focused tests, the 281-test green gate and three repeats of -36 cases passed. The original 333-test selection has 38 failing methods: -exactly the four repaired cases disappeared, with no additions. Phase 1's nine -test repairs are complete; no production code or client artifact changed. -Native public GeoHash conversion and all other missing conversions remain -backlog items. D038 records that checkpoint, before the LONG text-null decision. - -Iteration 2.23 accepts LONG-to-STRING/VARCHAR/SYMBOL in ordinary Sender. One -binding-local formatter uses existing target buffers; no server production or -send-time change is needed. `Long.MIN_VALUE` is an effective target-null write -in schema mode, with legacy behavior unchanged. Exact wire, shared vectors, -SQL, rollback, schema rebind and replay checks pass: 420 client tests plus two -packaging checks, 16 focused server tests, 288 server regressions and three -43-test seeded repeats. The original 333-test selection now has 35 errors: -exactly the three LONG-to-text cases disappeared, with no new failing method. -D039 records evidence and two stale rejection-oracle migrations. - -Iteration 2.24 accepts UUID-to-STRING/VARCHAR with the separately approved -both-limbs-MIN target-null rule. Every other pair, including either single MIN -limb, becomes canonical lowercase text. One lazy binding-local formatter feeds -the existing VARCHAR buffer; native UUID, legacy behavior and UUID-to-SYMBOL -rejection remain unchanged. Validation passes 426 client tests plus two packaging -checks, 21 focused server tests, 293 server regressions and three 48-test seeded -repeats. The original 333-test selection now has 33 errors: exactly the two -UUID-to-text cases disappeared, with no additions. Native UUID performance -checks found no material regression; they do not measure text throughput. D040 -records evidence and fixture corrections. Floating-point text-null rules were -still undecided at that checkpoint. - -Iteration 2.25 accepts FLOAT/DOUBLE-to-STRING/VARCHAR/SYMBOL with NaN as an -effective SQL NULL. Finite values, signed zero and infinities match the server; -FLOAT widens before formatting. A schema-only Ryu port preserves legacy Numbers. -Validation passes 432 client tests plus two packaging checks, 29 focused server -tests, 301 server regressions and three 56-test seeded repeats. The original -333-test selection has 30 errors: only the three DOUBLE-to-text cases disappeared. -Independent differential checks cover approximately two million non-NaN inputs. -Performance checks caught an inlining regression; separating text dispatch -removed the extra native DOUBLE instruction cost. D041 records the evidence, -remaining measurement limits and fixture corrections. Phase 2 is complete. - -Iteration 2.26 accepts LONG-to-ordinary-TIMESTAMP/TIMESTAMP_NS as raw counts in -the target unit. The existing LONG setter needs only two allowed targets and -two switch labels sharing its raw append. MIN remains an effective SQL NULL; -independent legacy frames confirm both bitmap contexts also store timestamp -NULL. Native LONG inference and named designated-timestamp rejection are -unchanged. Validation passes 437 client tests plus two packaging checks, -41 focused server tests, 305 regressions and three 68-test seeded repeats. -The original 333-test diagnostic has 29 errors: only LONG-to-TIMESTAMP was -removed from the prior 30. Native LONG measurements detect no regression; -they do not measure timestamp throughput. D042 records the evidence and a -corrected exact-wire fixture error. - -Iteration 2.27 accepts STRING-to-ordinary-TIMESTAMP/TIMESTAMP_NS. A fixed parser -preserves the server's eleven formats, UTF-8 replacement/ASCII-byte input view, -calendar arithmetic and timezone rules; nano targets check the multiplication -after microsecond parsing. Java-null input writes a bitmap null, while non-null -text that parses to MIN remains a present raw timestamp. No general compiler, -SQL-engine change, legacy change or new replay path is introduced. -Validation passes 442 client tests plus two packaging checks, 47 focused server -tests, 311 regressions and three 74-test seeded repeats. The original diagnostic -has 28 errors: only STRING-to-TIMESTAMP was removed from the prior 29. -The shared corpus has 110 literal cases; 214,030 public binding/encoder checks -per locale agree with the server in fresh en_US/tr_TR processes. Reference -century, locale and tzdb data still come from each process's runtime; this does -not guarantee agreement across different runtime data. Native STRING-to-VARCHAR -measurements show a consistent roughly 2–3% cycle/time cost, below the declared -5% guard, with effectively unchanged instructions. D043 records the findings, -corrections and measured trade-off. - -Iteration 2.28 accepts timestamp-to-STRING/VARCHAR. These writes reject -out-of-range values locally with INVALID_VALUE. NANOS are divided by 1000 toward -zero; MICROS through DAYS and Instant are converted to micros with range-correct -checked arithmetic before millisecond formatting. Instant keeps the existing -microsecond rounding. Preserve present MICRO MIN as empty text and NANO MIN as -finite timestamp text; this is not the LONG/NaN null policy. Old-server/legacy -behavior remains unchanged. One fixed formatter reuses the existing text sink; -no protocol, replay or SQL-engine change is needed. Validation passes 447 client -tests plus two packaging checks, 53 focused server tests, 317 regressions and -three 80-test seeded repeats. All 160,202 public binding/encoder comparisons -match the server formatter and checked-range oracle. The original 333-test -diagnostic now has 26 errors, removing only its two timestamp-to-text cases. -No repeatable native timestamp-write regression is detected by the performance -guard; this does not measure formatter throughput. D044 records the evidence. - -Iteration 2.29 accepted native CHAR and STRING-to-CHAR using existing two-byte -storage. Native values retain all 16 bits. Text takes the first BMP character, -ignoring the remainder; empty/NUL/supplementary-first text becomes code unit -zero, while malformed UTF-16 becomes '?', matching existing client replacement -and server decoding. Java null is bitmap-null; present zero remains wire-present -but reads as SQL NULL. CHAR-to-text remained separate at that checkpoint and -is implemented by iteration 2.47. Validation -passes 452 client tests plus two packaging checks, 59 focused server tests, -323 regressions and three 86-test seeded repeats. All 65,549 public differential -checks match the server. The unchanged original 333-test diagnostic has 24 -errors, removing only its two CHAR cases. The native string-write performance -guard passes. D045 records the corrected refresh catch and full evidence. - -Iteration 2.30 accepts STRING-to-LONG256 using four local longs and existing -32-byte storage. Match the server's lowercase `0x` prefix, 2–64 hex digits in -complete byte pairs and right-to-left limb order. Java null/omission is null; -the literal with four Long.MIN_VALUE limbs is INVALID_VALUE. Parse completely -before append. Native `long256Column` remains unsupported, and missing string -columns still infer VARCHAR. Validation passes 458 client tests plus two -packaging checks, 63 focused server tests, 327 regressions and three 90-test -seeded repeats. All 40,055 public binding/encoder differential checks match -the server oracle. The unchanged original 333-test diagnostic has 23 errors, -removing only STRING-to-LONG256. The native STRING-to-VARCHAR regression guard -passes; this does not measure LONG256 throughput. D046 records the evidence. - -Iteration 2.31 accepts STRING-to-GEOHASH using the existing base32 parser and -precision field. Parse at most 12 characters, require enough bits and truncate -to the target precision. Empty text and Java null become bitmap null. Always -emit the bitmap for schema-directed nullable GEOHASH columns, even with no null -rows: otherwise valid byte-aligned maximum values are mistaken for nulls. -Legacy and low-level non-nullable encoding remain unchanged. Only canonical -packed GEOHASH types are recognized; unknown flags retain typed rejection. -At that checkpoint native GEOHASH setters remained unsupported and missing -strings still inferred VARCHAR; iteration 2.39c completes the native setter -slice. Validation passes 464 client tests plus two packaging checks, 67 -focused server tests, 331 regressions and three 94-test seeded repeats. All -7,210 public differential cases match. The original 333-test diagnostic has -22 errors, removing only STRING-to-GEOHASH. Scoped native STRING-to-VARCHAR -checks measure 1.46–1.80% more append time and 0.17–1.15% more encoding time, -below the declared regression threshold; this is not GEOHASH throughput -evidence. D047 records the null-framing correction, flagged-type regression -and independent reviews. - -Iteration 2.32 accepts all three native decimal setters into every -DECIMAL8/16/32/64/128/256 target. This is 18 logical pairs represented by nine -physical source/target wire-width pairs: small decimal targets use DECIMAL64 -wire storage while retaining their own precision and scale. Conversion copies -the caller's value, rescales exactly, checks target precision and physical -storage width, and appends target-width limbs with the target scale retained -across reset. Public null setters remain no-ops; confirmed MISSING and legacy -paths keep source-width inference and legacy scale behavior. Tests cover caller -immutability, bitmap-only decimal nullness, exact wire/SQL results, failed-row -rollback and schema-generation transitions. Validation passes 472 client tests -plus two packaging checks, seven focused and 338 canonical server tests, and -three 101-test seeded repeats. The unchanged 333-test diagnostic has 18 errors: -only its four intended phase-5a methods disappeared. D048 records the 60-row -shared corpus and the bounded performance evidence. The 12-run legacy A/B guard -passes with +0.001% to +0.002% instructions and -0.47% to +1.99% cycles. Six -candidate-only nine-physical-pair runs measure median absolute cost of 32.91 ns, -182.03 cycles and 1140.18 instructions per value; they make no network, ingestion -or general-throughput claim. Phases 5b–5e remain; this does not establish complete -conversion coverage or release readiness. One low-priority diagnostic issue is -deferred: scale-up overflow has the correct `INVALID_VALUE` reason and rollback, -but its detail says the value cannot be rescaled exactly instead of saying that -it overflowed. Do not add another conversion branch solely to refine that text. - -Iteration 2.33 accepts LONG into every DECIMAL8/16/32/64/128/256 target. The -client treats the integer as a scale-zero value, rescales it exactly to the -target scale, checks declared precision and physical storage width, and writes -the direct target decimal wire type. `Long.MIN_VALUE` is a stable schema-mode -source null: it emits a decimal bitmap null and locks the target scale, regardless -of whether another row omits the column. Confirmed MISSING and legacy paths are -unchanged; the latter retains its existing bitmap-dependent sentinel behavior. -The implementation reuses the decimal column's existing scratch value and the -phase-5a validation/append tail; it adds no schema state, conversion registry or -send-time transformation. Validation passes 475 client tests plus two packaging -checks, seven focused and 345 canonical server tests, and three 108-test seeded -repeats. The unchanged 333-test diagnostic has 14 errors: exactly the four -LONG-to-decimal methods disappeared, with no additions. D049 records the 39-row -shared corpus, schema-generation test, compatibility characterization and -bounded performance evidence. The 12-run LONG-to-LONG A/B guard detects no -repeatable 5% cycles/instructions regression; this is not decimal, network or -ingestion throughput evidence. Phases 5c–5e remain, so release coverage is not -complete. - -Iteration 2.34 accepts STRING into every DECIMAL8/16/32/64/128/256 target. The -client parses at the target precision and scale with the server's `strict=false`, -`lossy=false` rules, including exponent underscores, then writes the QWP decimal -wire type for the target storage class through the existing decimal append tail; -DECIMAL8/16/32/64 targets share QWP's DECIMAL64 wire type. The shared -parser now uses the current server's leading-zero and full-scale precision -calculation. Exact, optionally signed `NaN` and `Infinity`, after supported -suffix processing, and Java null become target-scale bitmap SQL nulls. -Malformed special-value prefixes and extreme nonzero integer-boundary exponents -fail locally. -These rules deliberately do not copy three server defects: accepting -special-value prefix garbage as null, narrowing a special null to numeric zero -for small decimals, and throwing an array-bounds exception for extreme exponents. - -Confirmed MISSING columns and legacy mode remain VARCHAR and retain server-side -conversion. The implementation reuses the per-column Decimal256 scratch and the -phase-5a append tail; it adds no state, framework, protocol, server production, -SQL or send-time transformation. The 55-row shared corpus contains 19 values, -24 invalid inputs and 12 nulls; both copies have SHA-256 -`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`. -Client red validation first failed eight tests; final focused validation passes -eight tests plus two packaging checks, all-decimal validation passes 463 tests, -and the canonical gate passes 941. The installed candidate JAR has SHA-256 -`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. -A 3,601,944-case differential passes ordinary server parity and records 72 -intentional malformed-prefix corrections. Updating the shared parser's precision -metadata required correcting 19 stale assertions in 16 existing tests; their -values and scales did not change. - -The parser correction is global, not limited to schema mode: direct -`Decimal64`, `Decimal128` and `Decimal256` string parsing now uses the corrected -precision metadata, rejects malformed special-value prefixes and reports -extreme nonzero exponents as `NumericException`. This deliberate compatibility -change removes unsafe or undocumented behavior for legacy and standalone parser -callers as well as enabling schema-directed conversion. - -Server validation passes seven focused and 352 canonical tests. The unchanged -original 56-test conversion class now has ten errors: exactly the four -STRING-to-decimal -methods disappeared, with no additions. Three seeded repeats pass 115 tests and -22 archived XML reports each. A 12-run balanced fixed-work native -STRING-to-VARCHAR guard uses checksum `15640344951909530213`; paired basic -instructions change by -0.001% to -0.000% and cycles by -1.028% to -0.007%, with -no repeatable 5% regression. This is not decimal, network or ingestion throughput -evidence. D050 records the bounded acceptance. Phases 5d–5e remain, so release -coverage is not complete. - -Iteration 2.35 accepts FLOAT and DOUBLE into every -DECIMAL8/16/32/64/128/256 target. FLOAT is widened exactly to DOUBLE, then the -existing schema formatter produces the shortest round-tripping decimal and the -existing decimal parser enforces the target precision and scale without rounding -or truncation. Finite values whose natural scale or precision does not fit fail -locally with `INVALID_VALUE`; NaN and both infinities become target-scale bitmap -SQL nulls. Confirmed MISSING and legacy mode retain native FLOAT/DOUBLE bytes and -server-side conversion. - -The implementation adds only a decimal branch to the existing cold nonnumeric -floating helper and reuses the per-column Decimal256 scratch plus the phase-5a -target-width append tail. It adds no state, parser, framework, protocol, server -production, SQL or send-time transformation. Moving the branch out of the common -numeric helper was necessary: the first implementation pushed that method past -HotSpot's hot inlining threshold and increased native FLOAT/DOUBLE instructions -by about 3.8–4.7%. After the split, six counterbalanced pairs per native path show -median instruction changes of +0.0001% for FLOAT and +0.0037% for DOUBLE; median -cycle changes are -0.83% and -1.04%. This is a native binding regression guard, -not floating-to-decimal or ingestion throughput evidence. - -The shared client/server corpus has 60 rows: 37 values, 17 invalid values and six -nulls, with SHA-256 -`366a889719c6e7b688ff00e1bdfb95e273ab192508a1094dac5312c3d899611d`. -A 5,996,868-case differential matches the server's accepted values, target scale -and all four limbs. Client focused validation passes 63 tests plus two packaging -checks and the canonical gate passes 944; an expanded 1,269-test regression gate -also passes. The candidate and installed JAR SHA-256 is -`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. -Server validation passes the eight-test focused gate and 360-test canonical -gate. At that checkpoint, the unchanged original 56-test conversion diagnostic -had six errors, all assigned to phase 5e, with no additions. Three seeded repeats -passed 123 tests and 23 selected XML reports each. D051 records the test-fixture -cleanup and bounded phase-5d acceptance; the phase-5e acceptance below -supersedes its remaining-failure count. - -Iteration 2.36 accepts native Decimal64, Decimal128 and Decimal256 values into -STRING and VARCHAR. The binding promotes each non-null value into one lazily -reused Decimal256 scratch, applies the unsigned QWP scale byte, formats into the -existing reused text sink and appends `TYPE_VARCHAR`. This mirrors the server's -per-value scalar conversion, including full physical Decimal64/128 coefficients, -without a new validation policy, conversion registry, protocol state, server -change, SQL change or send-time transformation. Public null sentinels remain -setter no-ops. - -Schema mode intentionally formats each value independently, so mixed native -widths and scales can share a known text target. Legacy inference still fixes -one native width and scale for a whole column block and may reject or rescale a -later value. Preserving that irrelevant intermediate restriction after the text -target is known would add state and make conversion depend on neighboring rows. - -The 24-row shared corpus covers all six source/target pairs and both public null -forms. A second public boundary test covers the full physical Decimal128 value -`2^127 - 1` and unsigned scale-byte wrapping. The production/server differential -passes 3,000,294 full-physical and arbitrary-scale comparisons. Exact-current -client gates pass the 46-test public integration class, including four phase-5e -tests, plus 950 canonical tests, 1,275 broad tests and two packaged-JAR tests. -The candidate and installed JAR SHA-256 is -`e06759605ee3d1cf01c3277ba13531b0172e42138042c1f2f49c2db49e04b22d`; -only `QwpSchemaBinding.class` differs from the Phase-5d JAR. - -The unchanged server conversion diagnostic now passes all 56 methods, resolving -the original 47 failures. The accumulated server gate passes 363 tests; three -seeded repeats pass 126 tests each. The identity-path performance guard finds a -0.71–1.38% instruction increase and cycle medians from -1.93% to +1.13%, below -the 5% rejection threshold. It makes no speed, network or ingestion-throughput -claim. D053 and the iteration-2.36 run manifests record exact evidence. - -## Automatic negotiation and compatibility - -The new client works with old servers and automatically enables schema-directed -encoding when the server supports it. There is no configuration switch or -alternate write API to disable it on a supporting server. The earlier -`schema_encoding` and `schema_validation` options are removed from the proposal. - -Keep the base protocol at QWP v1: the existing magic, version byte and version -negotiation do not change. Negotiate the schema extension during WebSocket -upgrade using these proposed headers: - -``` -Client request: X-QWP-Request-Schema: true -Server response: X-QWP-Schema: enabled -``` - -The new client always requests the extension. A successful upgrade without the -response header selects legacy behavior, unless this sender has previously -confirmed schema support. Explicit confirmation enables schema requests, -schema-directed encoding and extended responses. An unrecognized response -header is a negotiation error, not evidence of an old server; neither are -connection failures, timeouts or authorization failures. Before any successful -upgrade, `ASYNC` may nevertheless choose the existing legacy encoding contract -for a new row. That is an explicit availability policy, not a claim that the -unreachable peer is an old server. - -The upgrade is one-way: once a sender confirms schema support, it assumes all -subsequent connections support the extension too. Require confirmation on -reconnect and background replay connections; never switch back to legacy mode. -Moving an upgraded sender to an old server is unsupported. Retain pending data -and report the capability mismatch under the existing reconnect policy. There -is no send-time transformation, metadata stripping or downgrade encoder. A -successful connection to an old server does not permanently prevent upgrade: -a later connection that confirms schema support adopts schema mode at the next -batch boundary: the first row after the pending legacy batch flushes. - -| Client | Server | Behavior | -|---|---|---| -| Existing client, no schema request | New server | Existing QWP v1 framing and behavior | -| New client, not yet upgraded | Old server | Existing inferred-type QWP v1 behavior | -| New client, `ASYNC`, no completed handshake | Unknown server | Best-effort legacy encoding until a handshake selects the mode | -| New client, always requests schema | Supporting server | QWP v1 with the schema extension; one-way upgrade | -| New client, already upgraded | Old server | Unsupported endpoint; retain pending data, no downgrade | - -The server enables the extension only when requested and confirmed. Connections -without negotiation receive no schema frames or extended ACKs/NACKs. Existing -clients need no changes. Clients may be upgraded before servers. Once schema -support has been confirmed, failover destinations must also support it; mixed -old/new endpoints are not supported for that sender after the upgrade. - -A pending legacy batch finishes in legacy mode. A batch has one wire contract, -selected by its first committed row, because a QWP frame has one wire family: -while the batch holds legacy rows, later rows stay legacy. The producer adopts -schema mode on the first row after that batch flushes (explicitly or by -auto-flush) and replaces the inferred layout before binding new rows. Legacy -and schema blocks therefore never share a flush. Published legacy frames retain -their original types and flag-clear framing, even if sent later. A -supporting server accepts those flag-clear blocks on a schema-negotiated -connection; negotiation is required only for schema-flagged data. - -The extension defines one stable conversion contract, backed by shared test -vectors, rather than a separate conversion-version handshake or a dependency on -a particular server release. A schema alone does not specify parsing or -rounding rules. In schema mode, unknown target types fail explicitly when used, -without blocking writes to unrelated supported columns or falling back to -raw-value encoding. Schema lookup failures never disable schema mode. - -## Availability - -Reuse the existing initial-connection policy; do not add a schema-specific mode -or option: - -- `OFF` makes one foreground connection and negotiation attempt. Failure aborts - construction. -- `SYNC` retries connection and negotiation on the caller thread within the - configured reconnect budget. Exhaustion aborts construction. -- `ASYNC` returns before connecting. Until the first successful handshake, a - row whose first effective operation finds no negotiated mode is accepted - immediately under the existing legacy conversion and flag-clear framing. - This is best-effort schema use, not deferred validation: the client never - retains the original values for later conversion. - -The mode is pinned by the first effective operation. If the handshake completes -mid-batch, that batch remains legacy and schema mode can start only on the first -row after it flushes. -Consequently, handshake timing can change successful stored values as well as -error timing: the legacy and schema contracts deliberately differ for cases -such as sub-microsecond timestamps, overflow and source-null normalization. -Release notes and API documentation must describe this as legacy conversion -before confirmation, not merely skipped validation. - -The `ASYNC` exception applies only before the first successful handshake and -only when recovery has not already found schema-framed data. A recovered -schema-required backlog must negotiate schema support before accepting new -data; it must not append speculative legacy rows. If the first successful -server is old, rows remain legacy until a later connection confirms schema -support. Once support is confirmed, the sender never falls back to legacy, -including on reconnect, cache miss or lookup failure. - -After schema support is confirmed, required lookups use a fixed 30-second -deadline. A cache-miss describe or forced refresh starts its own 30-second -budget. This is not a public configuration surface. Timeout, unavailable -metadata or request saturation throws a lookup error before accepting the -value. It cancels any partial row but does not permanently halt the sender. -Recoverable transport loss resends the pending lookup after reconnection within -its original deadline. Do not retry completed lookup errors or add separate -timeout settings for lookup paths. - -### Minimal client integration - -Change only `QwpWebSocketSender.bindingForEffectiveWrite()`. Keep its existing -in-progress-row fast path first, then ensure the existing I/O loop is started. -Before entering the current deadline, negotiation and lookup path, select -legacy behavior with this condition: - -```java -if (initialConnectMode == Sender.InitialConnectMode.ASYNC - && !cursorEngine.requiresSchema()) { - schemaResolutionFresh[0] = false; - return null; -} -``` - -`CursorSendEngine.requiresSchema()` is the existing sticky authority. It is -false for a fresh asynchronous sender and after a successful old-server -handshake. It becomes true before publication after a supporting handshake, -and recovery initializes it to true when persisted frames require schema -support. That single value therefore covers startup, old-server compatibility, -one-way upgrade and recovery. Do not also consult `hasEverConnected()` and do -not add another negotiated-mode field. - -A supporting handshake may set the requirement immediately after the producer -observes false. That race is harmless and intentional: the current batch has -already selected legacy behavior, while the first row of the next batch observes -the sticky true value and enters schema mode. One producer-thread flag, set by -the batch's first committed row, preserves that boundary without a lock, and no -flush ever mixes wire families. - -No change belongs in `QwpSchemaCoordinator`, `CursorWebSocketSendLoop`, -`CursorSendEngine`, the encoder, store-and-forward format or server. Update the -`ASYNC` API documentation and the outdated method comment that currently says -an unavailable first connection can never select legacy encoding. - -Known schemas can be used offline; unfamiliar tables require a live lookup. -Reconnect invalidates the lookup cache for future row starts, but does not reset -an upgraded sender to legacy mode, invalidate an already pinned row or rewrite -buffered and persisted blocks. Legacy mode and the pre-handshake `ASYNC` -exception retain existing offline buffering. A bad speculative legacy row may -therefore be rejected only after connection and can stop the persisted queue -under the existing terminal-NACK policy; metadata cannot repair or reconvert -it. This is the cost of choosing asynchronous startup. - -## Frame layout and replay compatibility - -Negotiation authorizes the extension; a persisted frame flag identifies the -actual data layout. Use `FLAG_SCHEMA = 0x40` without changing the QWP version byte -(`0x02` belongs to the server's `FLAG_DURABLE_ACK_POLL`): - -- Flag set: each table header includes its pinned schema identity, or an - explicit unknown identity. The server uses the extended table-header parser. -- Flag clear: table headers have the existing layout and no schema identities. - These are legacy-mode writes or legacy SF frames, accepted even on a - schema-negotiated connection. -- Flag set without successful negotiation: an extension-aware server rejects - the frame before processing table data. The client must never send it to an - endpoint that did not confirm support. - -In a flagged frame, each table header starts with the existing name-length -varint and UTF-8 name, followed by `identityKind:u8`. Kind 0 means unknown and -has no body. Kind 1 carries `tableId:i32 LE` and `metadataVersion:i64 LE`, both -non-negative. The existing row-count and column-count varints and column -definitions follow. Reject reserved kinds, truncated identities and negative -known fields. Flag-clear headers retain their exact old layout. - -`FLAG_SCHEMA` cannot be combined with `FLAG_CONTROL`, and schema-flagged frames -must contain at least one table. Table-less commit/dictionary traffic stays -flag-clear. The parser still uses inline column definitions to interpret the -values; schema identities never replace them or enforce a version precondition. - -Every frame containing schema-mode table blocks sets the flag, including writes -that use auto-creation. Legacy blocks keep flag-clear framing. Each frame uses -one table-header layout; keep legacy and schema-mode blocks in separate frames -during the one-way transition, preserving order and transaction boundaries. -Table-less dictionary/control traffic retains its own framing rules. - -Persist the flag with the schema identities and data bytes. Replay preserves -all of them, allowing legacy and extended data frames to coexist in sequence on -one negotiated QWP v1 connection. Do not infer the table-header layout solely -from connection state or relabel old frames with new flags or identities. - -Repeat negotiation on every reconnect, including background replay connections. -Before upgrade, legacy SF can drain to either an old or a supporting server. -After upgrade, require support even when the remaining frames happen to be -legacy. Recovery of pending extended SF data also requires support: establish -that requirement before replaying any part of that backlog, including a legacy -prefix. A fresh sender with only legacy SF negotiates normally. An incompatible -endpoint cannot drain a schema-required backlog: retain pending bytes and find -or wait for a compatible endpoint under the reconnect policy. Do not advance -watermarks or alter stored frames. - -ACK/NACK extension framing is selected by the negotiated connection, independent -of the data frame's flag. A negotiated connection may receive schema feedback -for legacy blocks too; their absent identities are treated as unknown. Ordinary -and durable-ACK delivery semantics remain unchanged. - -Keeping QWP at v1 does not make extended SF data readable by an older client. -Before downgrading the client binary, stop new writes and drain all pending -extended data, including background/orphan slots, with a supporting client and -server. Completion means acknowledgment under the configured ordinary or -durable-ACK policy, not merely a successful send or an attempted close. -Downgrading with pending extended data is unsupported. If draining fails, -retain the data and continue recovery with a supporting client; do not rewrite -frames, strip metadata, delete pending logs or advance watermarks to permit the -downgrade. This is an operational prerequisite, not a claim that older binaries -can detect and enforce it. - -## Schema lookup - -Each schema carries `(tableId, metadataVersion)`. The server already versions -table metadata; `tableId` distinguishes drop/recreate. The pair is an identity, -not a global ordering between table incarnations. - -The logical control exchange is: - -``` -DESCRIBE: requestId, tableName -SCHEMA: requestId, tableName, result, schema? -schema: tableId, metadataVersion, designatedIndex, columns[] -column: name, serverType, typeParams -``` - -The schema includes full target types and parameters: decimal precision/scale, -timestamp units, array element type/rank and geohash bits. -`designatedIndex` identifies the target of `at(ts)`; -1 means none. -Names follow the server's comparison rules. There is no per-column -`acceptMask`: the extension's conversion contract and target type determine -the client converter. - -Results distinguish a known schema, a confirmed missing table and unavailable -metadata. Missing tables may follow auto-creation; unavailable metadata must -never be represented as a missing table. Permission denial has a distinct -machine-readable reason and must not trigger auto-creation. - -An account authorized to write a table must be able to obtain the ingestion -schema needed for that write without gaining SELECT access to its data. Apply -the table-write authorization to both DESCRIBE and schema feedback in ACKs/NACKs; -metadata being available is not sufficient permission to return it. Return only -the ingestion schema, not table data or unrelated privileged metadata. Retain -the existing authorization rules for creating missing tables and adding columns. -Verify write-only accounts and denial/revocation behavior as part of the server -lookup implementation. Cached metadata never substitutes for server-side write -authorization. - -The producer waits while the I/O thread services the connection and lookup. -Describes do not flush rows, consume data-frame sequence numbers or affect -ACK watermarks. - -Direct SCHEMA replies retain the full QWP control-frame header. Route that -envelope separately from ordinary and durable ACKs, then validate it strictly; -do not guess the response type by trying multiple decoders. The request ID -identifies the requested table; the current direct reply does not repeat its -name. Named ACK/NACK feedback reuses only the inner schema payload. - -## Column binding and conversion - -This section describes schema mode. Legacy mode keeps existing binding, -conversion and missing-value behavior unchanged. - -The sender maintains a sparse layout containing only columns being written. -For each known column, its binding contains the target type and parameters, -the chosen QWP wire representation, and converters for the supplied input -types. Schema resolution and converter selection can be cached; conversion -and value-dependent checks happen on every effective write. - -Server types and wire types are not necessarily one-to-one. For example, a -small server DECIMAL may use a DECIMAL64 wire representation. The client must -still enforce the server column's precision and scale before buffering. - -For text and symbol inputs, STRING and VARCHAR targets both use the existing -VARCHAR wire representation; a SYMBOL target uses symbol encoding. Reuse the -existing UTF-8 serialization rules: preserve case, whitespace, embedded NUL and -valid Unicode, and replace each unpaired UTF-16 surrogate with literal `?`. -Null and empty text remain distinct. Symbol dictionary IDs are representation, -not value identity: distinct source strings may encode to the same text after -surrogate replacement. No new dictionary owner or send-time type conversion is -needed for these text-to-text pairs. - -For a BINARY target, binary setters copy opaque bytes without interpretation. -`stringColumn` writes the same UTF-8 bytes it would send through the legacy -VARCHAR representation, including NUL and the surrogate-replacement rule above; -this is not hex or base64 parsing. All three binary input forms (byte array, -DirectByteSlice and native pointer/length) select TYPE_BINARY. Effective null -byte-array/slice arguments report `INVALID_VALUE`; string null and omission -produce target NULL. Empty input remains a present zero-length BINARY value, -distinct from NULL on the current server. - -Native binary lengths must be 0..Integer.MAX_VALUE, with a nonzero pointer for -nonempty input. Reject invalid lengths before reading memory. The caller must -provide readable memory for the duration of the setter; the binding retains -neither the source address nor the slice. A zero pointer with zero length is -valid empty input. First-value-wins precedes these checks. Binary input into -other target types is not implemented, including the unresolved parser paths -listed below. - -String and binary byte appends preserve their previous data and offset positions -if a RuntimeException or Error occurs before publication. The same exception is -rethrown; the caller still cancels the partial row and rolls back its new column -definitions. This repairs exceptional cleanup in the shared buffer, not just -schema mode, without changing successful wire bytes. It does not claim recovery -from invalid native addresses or fatal process faults. - -Boolean inputs use the target representation: BOOLEAN values, numeric `0`/`1` -for BYTE/SHORT/INT/LONG/FLOAT/DOUBLE, and lowercase `false`/`true` text for -STRING/VARCHAR. BOOLEAN-to-SYMBOL is not a supported server conversion. -Text-to-BOOLEAN accepts only `0`, `1` and ASCII-case-insensitive `true`/`false`. -Do not trim whitespace, apply Unicode case folding or silently treat an invalid -token as false. Invalid text reports `INVALID_VALUE` before append. -Explicit null and omission follow the target rule below: BOOLEAN stores false, -BYTE/SHORT store zero, and nullable numeric/text targets store NULL. This does -not make unsupported input/target pairs valid. - -Unlike the current inferred layout, a known column is not bound to the first -setter's input type. Different setters may feed the same target buffer across -rows, provided their conversions are supported. Existing name validation and -setter no-op rules remain. In schema mode, first-value-wins is applied before -conversion even when the duplicate uses a different setter; ignored values -are not converted. Legacy mode ignores same-type duplicates but throws on -cross-setter type mismatches. - -Omitted ordinary columns and explicit nulls use the target column's missing-value -representation: SQL NULL where supported, otherwise the target's native -missing-value/default representation. Do not synthesize a source-type default -and then convert it. A setter that treats null as a no-op still does not bind or -write a value; any resulting omission follows the target rule. Inferred columns -use their inferred target type until a server-backed binding replaces it. - -This intentionally changes some existing results. After a boolean setter binds -the old client layout, an omitted value sent to a STRING column currently becomes -the text "false". With a STRING target binding, omission instead becomes NULL; -an explicitly supplied false still converts to "false". Missing-value behavior -must not depend on which input setter was used in an earlier row. The designated -timestamp remains the documented exception: `atNow()` uses server-assigned time. -Document this behavior change and test both negotiated modes and old clients. - -For LONG input to a numeric target in schema mode, `Long.MIN_VALUE` means null -before conversion, regardless of other rows in the block. Append the target's -missing-value representation: zero for BYTE/SHORT, NULL for INT/LONG/FLOAT/DOUBLE. -Apply duplicate suppression and validate the supported input/target pair first; -this rule does not make an unsupported conversion such as LONG-to-UUID valid. - -For LONG input to STRING, VARCHAR or SYMBOL, apply the same source sentinel -decision: `Long.MIN_VALUE` writes target SQL NULL before formatting, independently -of other rows. Other values become canonical signed decimal text. This approved -schema-mode rule deliberately differs from legacy bitmap-present MIN, which -can become the literal text "null". An explicit MIN is an effective first write, -not a no-op; an ignored duplicate cannot replace it. - -This deliberately differs from legacy QWP's block-dependent sentinel handling. -With a null bitmap present, a supplied LONG minimum can be rejected by a narrow -integer target or stored as a finite FLOAT/DOUBLE; without the bitmap, it is -treated as null. Schema mode does not reproduce that inconsistency. Legacy -writes and existing legacy SF retain their behavior. No end-of-batch or -send-time conversion is introduced to imitate it. - -Conversion parity with QWP ingestion for supplied values is a release -requirement, except for the deliberate missing-value and LONG source-null rules -above and the LONG text range, floating-point and timestamp rules below. -Coverage includes: - -- String parsing into UUID, numbers, timestamps, decimals and other supported - targets; conversions to text must preserve the server's formatting rules. -- Numeric range, fractional-value, precision-loss, rounding, null-sentinel - and special floating-point behavior. -- Timestamp unit conversion and overflow; decimal precision and rescaling. -- Each non-null array's supported element type and rank, including wrapper and - Java overloads. Rank is not fixed by today's client layout. Do not promise - element conversion: the server copies array elements, and its exposed - `LONG_ARRAY` path has unresolved validation gaps described in the backlog. -- Geohash precision: native GEOHASH input requires an exact bit-count match, whereas - string parsing follows the server's geohash conversion rules. - -Do not substitute a convenient library parser or Java cast without proving -equivalent behavior. Use shared conformance vectors across server and clients, -checking acceptance, rejection and resulting stored values. An implementation -with missing converters must report that limitation, not claim a schema -mismatch or silently pass the value through to the server. - -For STRING input to LONG in schema mode, reject text outside the mathematical -signed-long range before append. This is an approved compatibility exception: -legacy ingestion accepts some overflowing decimal strings and stores wrapped -values, for example `"21000000000000000000"` becomes `2553255926290448384`. -Legacy parsing and ingestion retain that behavior. Preserve the server's other -LONG text grammar; do not use this range fix to broaden or tighten syntax. -An accepted string representing LONG minimum remains a supplied target -sentinel that reads SQL NULL, not a LONG setter's source-null conversion. - -For FLOAT/DOUBLE input to supported numeric targets, NaN means source null, -independent of other rows. Apply duplicate suppression and validate the supported -input/target pair first, then append the target's missing-value representation. -NaN payload and sign bits are not retained as values. Integer targets require a -whole value within the target's mathematical range; fractions, infinities and -out-of-range values report `INVALID_VALUE` before append. The LONG upper bound -is exclusive positive `2^63`, not a floating-point approximation of -`Long.MAX_VALUE`. Exact negative `2^63` and INT minimum remain representable -values that read as the target's SQL-null sentinel, matching server behavior. - -These are approved schema-mode exceptions: legacy NaN-to-integer writes use -the target's missing value without a null bitmap but fail when another row's -omission causes a bitmap; legacy positive `2^63` to LONG silently stores -`Long.MAX_VALUE`. Legacy ingestion and replay keep those behaviors. FLOAT/DOUBLE -targets still follow server widening/narrowing, including signed zero, -infinities, subnormal rounding and finite DOUBLE overflow to FLOAT infinity. -Do not add batch-dependent or send-time conversion. - -For typed microsecond/nanosecond inputs into ordinary timestamp columns, -nanoseconds-to-microseconds division truncates toward zero, including negative -values. Microseconds-to-nanoseconds multiplication rejects values outside the -server's representable range before append. Matching units preserve the bits. -The primitive timestamp input does not inherit the LONG-input null rule: -`Long.MIN_VALUE` overflows when scaling micros to nanos, becomes a finite value -when dividing nanos to micros, and reads as SQL NULL when stored unchanged. -Explicit nulls and omissions still use the target's null representation. -Test these cases both with and without another omitted row in the block. - -For primitive timestamp inputs, support NANOS, MICROS, MILLIS, SECONDS, MINUTES, -HOURS and DAYS. Convert directly to the target unit with checked arithmetic; -values outside its range report `INVALID_VALUE` before append. Other units -report `UNSUPPORTED_FEATURE`, and a null unit reports `INVALID_VALUE`. - -For `Instant`, preserve all nanoseconds in a TIMESTAMP_NS target. For TIMESTAMP, -retain the existing formula `epochSeconds * 1000000 + nanoFraction / 1000`; -this differs from primitive nanos-to-micros truncation for negative sub-micro -instants. Check the final combined value's range: an intermediate multiplication -must not reject an otherwise representable negative instant. A null `Instant` -reports `INVALID_VALUE`, not an implicit timestamp null. Duplicate suppression -precedes argument validation and conversion for both overloads. - -These are intentional schema-mode changes: legacy QWP scales larger units with -unchecked arithmetic and converts `Instant` to micros even for a nano target. -Legacy mode keeps that behavior. Shared vectors must test the new contract and -explicit legacy comparisons must document the changed overflow and precision. - -`at(ts)` converts to the pinned designated timestamp's unit before append and -row commit, including when the timestamp column reference is cached. A known -schema without a designated timestamp rejects explicit `at(ts)`. -`atNow()` resolves a schema if no earlier effective setter did so, then keeps -server-assigned timestamp behavior. - -Confirmed missing tables or columns retain client-side inference and existing -auto-create policy. Inferred bindings are distinct from server-backed bindings; -the client cannot promise conversion validity against a target that does not -yet exist. Successful writes return the resulting schema. - -Inference uses the existing buffer's column definitions, not fabricated server -metadata. The first effective supported setter establishes the native type: -BOOLEAN, LONG, FLOAT, DOUBLE, VARCHAR for text, SYMBOL, UUID, BINARY, CHAR, or a -timestamp. Primitive NANOS infers TIMESTAMP_NS; other supported units and -`Instant` infer TIMESTAMP. Only a missing table may infer the unnamed designated -timestamp. A conflicting type in a later row is a schema rejection and gets -the same one-refresh opportunity as a known-column rejection; a duplicate -within the current row is ignored. Failed-only definitions are removed by the -existing row rollback. Unsupported setter families remain unsupported even -when the table or column is absent. - -## Conversion backlog (server-source audit) - -Cross-checked on 2026-09-11 after iteration 2.13, with implemented coverage -refreshed through iteration 2.45 on 2026-09-14. The pre-iteration baseline is -client `47a37a6f` and server `c1f970220f`, whose submodule pins that exact -client revision. The original audit bases were client -`981bdb02a471f3b290c89b8e78cbc422610e329e` and server -`12a33d651e51e2682e7a448c8db5168fc72dfad3`. The matrix is a source audit; -runtime evidence for implemented slices is recorded separately in the journal. -Read the actual QWP cursor selection, target dispatch and -value routines together; neither a shared switch branch nor an SQL cast proves -that QWP accepts a pair. - -### Complete input-to-target inventory - -The table covers effective, non-null values into existing **ordinary** columns. -"Implemented" records `QwpSchemaBinding` coverage; Sender integration is -described separately below. -An accepted null, ignored duplicate or no-op does not establish support for a -non-null conversion. Known compatibility exceptions above still apply. - -- **Numeric** means BYTE, SHORT, INT, LONG, FLOAT and DOUBLE, separately. -- **Text** means STRING and VARCHAR, separately; SYMBOL is always listed apart. -- **Timestamps** means TIMESTAMP (microseconds) and TIMESTAMP_NS (nanoseconds), - separately (`TIMESTAMP_NANO` in server code). -- **Decimals** means every DECIMAL8/16/32/64/128/256 target, with its declared - precision and scale. **Geohash** means every valid 1..60-bit target. -- Input names refer to public setters. `stringColumn` emits `TYPE_VARCHAR`; - there is no separate `TYPE_STRING` wire code. `symbol` emits dictionary-backed - `TYPE_SYMBOL`, not text-parser input. BYTE/SHORT/INT setters emit their own - wire types; they are not aliases for `longColumn`. - -| Public input (source wire type) | Implemented targets | Missing server-accepted targets | -| --- | --- | --- | -| `boolColumn` (BOOLEAN) | BOOLEAN, numeric, text | None | -| `byteColumn`, `shortColumn` (BYTE, SHORT) | Numeric, DATE, timestamps, text, SYMBOL, decimals | None | -| `intColumn` (INT) | Numeric, DATE, timestamps, text, SYMBOL, decimals, IPv4 | None | -| `longColumn` (LONG) | Numeric, text, SYMBOL, DATE, timestamps, decimals | None | -| `floatColumn`, `doubleColumn` (FLOAT, DOUBLE) | Numeric, text, SYMBOL, decimals | None | -| `stringColumn` (VARCHAR) | BOOLEAN, numeric, text, SYMBOL, UUID, BINARY, DATE, timestamps, CHAR, LONG256, geohash, decimals | None | -| `symbol` (SYMBOL) | Text, SYMBOL | None | -| `timestampColumn(long, unit)` (TIMESTAMP or TIMESTAMP_NANOS) | Timestamps, text | None | -| `timestampColumn(Instant)` (currently TIMESTAMP micros in legacy Sender) | Timestamps, preserving nanos for the schema-mode nano target; text | None | -| `uuidColumn` (UUID) | UUID, text | None | -| `charColumn` (CHAR) | CHAR, text for non-surrogate values | None | -| `ipv4Column(int)` and `ipv4Column(CharSequence)` (IPv4; text parsed locally) | IPv4, text | None | -| `long256Column` (LONG256) | LONG256, text | None | -| `binaryColumn` (byte array, `DirectByteSlice`, native pointer/length; BINARY) | BINARY | Parser-reachable targets remain deferred; see boundaries below | -| `decimalColumn(Decimal64/128/256)` (DECIMAL64/128/256) | All decimals from each source width, text | None | -| `decimalColumn(CharSequence)` (locally parsed DECIMAL256) | All decimals, text | None; this is not the `stringColumn` parser path | -| `geoHashColumn(long, bits)` and `geoHashColumn(CharSequence)` (GEOHASH with source bits) | Geohash with exactly matching bit precision, text | None | -| `doubleArray` (all Java ranks and `DoubleArray`; DOUBLE_ARRAY) | DOUBLE array of exactly matching rank | None | -| `longArray` (all Java ranks and `LongArray`; LONG_ARRAY) | None | No supported conversion contract established; see the existing-server gap below | - -DATE also exists as a wire input, but there is no public WebSocket `dateColumn` -setter. The server accepts DATE-to-DATE and DATE-to-text only. Neither has a -binding entrypoint today; record this wire-only coverage without adding a new -public API just to mirror the protocol. DATE **targets** from existing integer -setters are implemented in iteration 2.40; STRING-to-DATE is implemented in -iteration 2.44. Ranked DOUBLE-array identity is implemented in iteration 2.45. - -### Conversion rules - -These are server-source requirements, not permission to silently add new casts -or copy the unresolved behaviours below. - -- **Integer sources:** retain distinct BYTE/SHORT/INT/LONG input contracts. - Numeric narrowing checks range. DATE takes raw epoch milliseconds; ordinary - timestamps take raw counts in the target's unit, with no implicit unit - conversion. INT-to-IPv4 has its own null translation; it is not LONG-to-IPv4. - Integer-to-text/SYMBOL uses `Numbers.append(long)`. -- **Floating output:** text/SYMBOL formatting uses - `Numbers.append(cursor.getDouble())`, including widening FLOAT first. Do not - substitute `Float.toString`. Test signed zero, infinities, NaN and rounding. - The current server uses Ryu while the client still has the older formatter; - matching method names and default scale do not establish identical text. - Iteration 2.25 implements an isolated server-equivalent Ryu formatter; - the shared legacy formatter is unchanged. -- **Text parsing:** DATE uses `DateFormatUtils.parseDate`; timestamps use - `MicrosFormatUtils.parseTimestamp`, then checked multiplication by 1000 for - a nano target. Do not assume a separate nanosecond text parser. LONG256 uses - `Numbers.parseLong256`. VARCHAR-to-BINARY preserves the existing UTF-8 byte - payload, not hex/base64 decoding. CHAR takes the first decodable BMP character - and ignores the remainder; empty, malformed raw UTF-8 or supplementary-first - input produces NUL. Public malformed UTF-16 instead becomes '?', matching - existing client replacement before server decoding. Java null uses bitmap - null; present NUL is distinct on the wire but also reads as SQL NULL (D045). -- **Timestamp output:** both text targets use - `MicrosFormatUtils.appendDateTime`; nano wire values divide by 1000, truncating - toward zero, before formatting. Cover each primitive unit and `Instant` - separately. D044 separately approves checked normalization into legacy - source precision for text targets, without changing legacy mode. -- **Other text output:** UUID uses `Numbers.appendUuid`, LONG256 uses - `Numbers.appendLong256`, DATE uses `DateFormatUtils.appendDateTime`, and - decimal output preserves source scale. The server formats IPv4 through - `Numbers.intToIPv4Sink`; iteration 2.39a uses the client's private reusable - `formatIPv4` sink. CHAR uses the server's `putAscii` path; see the non-ASCII - limitation below. -- **Decimals:** all three decimal wire widths reach every decimal target. - Decimal input rescaling must be exact: discarded nonzero digits cause - precision loss; scale-up/precision/storage overflow fails. Integer input is - scaled to the target; text is parsed at target precision and scale. - FLOAT/DOUBLE uses the server's `Numbers.doubleToDecimal(..., false)` path; - non-finite input becomes decimal null, and finite conversion overflow fails. - Do not replace its rounding with an assumed BigDecimal policy. Small server - decimals still need a supported wire width, such as DECIMAL64, while enforcing - the smaller target's precision/scale. All typed and text overloads need tests. - Iteration 2.32 implements and validates the three typed overloads into all six - decimal targets. Iteration 2.33 adds LONG into all six targets, treating - `Long.MIN_VALUE` as a target-scale bitmap null in schema mode. Iteration 2.34 - adds STRING into all six targets at the target precision and scale, preserving - exponent underscores and applying the stable special-value null rules above. - Iteration 2.35 adds FLOAT/DOUBLE into all six targets through the server's - lossless shortest-decimal path, with non-finite values encoded as decimal - nulls. Phase 5e converts non-null `Decimal64`, `Decimal128` and `Decimal256` - values targeting STRING or VARCHAR to plain fixed-point text before append. - Like the server, it first promotes the physical value to a reusable - `Decimal256`, applies the unsigned one-byte QWP scale, and then formats it. - This preserves valid source scales and trailing zeros, handles the full - physical Decimal64/128 range without the narrower formatters' precision - limit, emits `TYPE_VARCHAR` for both targets, and keeps public decimal null - sentinels as setter no-ops. - MISSING and legacy mode retain their native decimal wire type and source scale. - Other integer widths remain. - - Parity here is per setter value after schema binding, not a replay of the - legacy native-decimal column buffer. For a known text target, each row is - formatted independently and mixed Decimal64/128/256 inputs and scales are - accepted. Legacy inference instead fixes one native decimal width and scale - for the whole column block, rejecting a later width and rescaling or rejecting - later scales. This is intentional: once the target is known to be text, - preserving an irrelevant intermediate native-decimal layout would add state - and make valid per-value conversion depend on neighboring rows. -- **IPv4:** the packed zero sentinel becomes a bitmap NULL for IPv4 and text - targets, so its result cannot depend on whether another row creates a bitmap. - Text is parsed locally after target selection and duplicate suppression. - Preserve the legacy dotted grammar: case-insensitive `"null"` and exact - `"0.0.0.0"` are rejected, but dotted aliases that parse to zero normalize to - NULL. Both STRING and VARCHAR receive target-native VARCHAR wire data; no - server-side second conversion is required. -- **LONG256:** four `Long.MIN_VALUE` limbs are the source null sentinel in - schema mode and become an explicit bitmap NULL for LONG256 and text targets; - a partial match remains a value. LONG256 keeps the four little-endian limbs. - STRING and VARCHAR receive direct VARCHAR wire data formatted as lowercase - `0x` text with the most-significant nonzero limb first, even-byte leading - width and 16 hex digits for each lower limb. Legacy native wire behavior is - unchanged, including its bitmap-dependent server-side text conversion. -- **Geohashes:** wire precision must equal target bits; no native GEOHASH narrowing is - accepted. The packed overload masks bits above its declared 1..60-bit - precision. The textual GEOHASH overload retains its public contract: strict - base32, 1..12 characters, with five bits per character; null and empty values - are invalid, and omission writes NULL. Every packed bit pattern is a value. - Schema-mode GEOHASH buffers therefore carry the null bitmap so byte-aligned - all-one values cannot collide with the legacy cursor sentinel. Native - GEOHASH-to-text emits exactly one `0` or `1` per source bit. `stringColumn` - uses `GeoHashes.fromStringTruncatingNl`: empty means - null, parse at most the first 12 base32 characters, require enough bits and - truncate to target precision. Invalid consumed characters fail; characters - beyond the 12-character limit are ignored. The public geohash text setter - first produces a precision-bearing GEOHASH and must not be confused with - `stringColumn`. Geohash-to-text emits bit strings for valid positive wire - precision, not the helper's unreachable negative-precision character branch. -- **Arrays:** DOUBLE_ARRAY copies raw 8-byte elements; the canonical encoded - target type supplies both DOUBLE element type and strong rank. Rank must match - exactly, while per-row lengths may vary. A confirmed missing column pins the - first effective rank across rows and batches. Public null array references are - no-ops; omission produces SQL NULL and an empty shape is a value. The cursor - validates rank 1..32, nonnegative dimensions, checked element count and exact - payload size. No implicit LONG/DOUBLE element cast or rank conversion is - established by the server. - -Every implemented pair needs local rejection/rollback, exact target-wire and -stored-value checks, relevant recovery coverage, and explicit legacy comparisons -using shared vectors. In particular, the fixed-width server cursor treats a null -bitmap as authoritative and otherwise recognizes type-specific sentinels. -BYTE/SHORT/CHAR have no source sentinel; INT minimum, LONG minimum, floating -NaN, IPv4 zero and UUID/LONG256 sentinel limbs can behave differently when a -companion omitted row introduces a bitmap. Test both contexts. The approved -LONG/FLOAT/DOUBLE source-null rules currently specify numeric targets; extending -those rules to missing target families or new input types needs an explicit -contract, not an assumption that every legacy cast has stable null behaviour. - -Decimal cursors are different again: only bitmap bits mark null; raw decimal -sentinel limbs still enter server conversion as supplied values without a bitmap. -Iteration 2.32 characterizes that distinction while preserving public typed- -decimal sentinel/null no-ops. Same/different widths, exact rescaling, explicit -nulls and raw sentinel payloads are covered for decimal targets. For phase 5e, -valid raw native-decimal frames remain a legacy server-formatting check. Public -low-level constructors and `setScale` mutators can also expose coefficients or -scales outside the documented 18/38/76-digit domains. Schema mode does not add -a second validation policy for those values: it uses the same Decimal256 -promotion and unsigned one-byte scale that the server sees on the wire. Public -decimal null sentinels remain setter no-ops, so the server's non-bitmap sentinel -behavior is only a raw-frame boundary. - -### Server boundaries requiring explicit decisions - -These paths are not silently omitted from the inventory and are not newly -approved compatibility exceptions. Characterize them with public tests and -decide the contract before activating the affected Sender paths; copying an -unsafe behaviour merely because it is reachable is not the recommendation. - -1. **BINARY reaches text parsers.** BINARY and VARCHAR share - `QwpStringColumnCursor`. The BOOLEAN, numeric, DATE, timestamp, decimal, - geohash, UUID and LONG256 target branches do not exclude BINARY, so suitable - binary bytes reach those parsers. All those pairs are missing in the binding. - In contrast, CHAR/STRING/VARCHAR/SYMBOL explicitly reject BINARY. Decide - whether binary-to-parser pairs are supported compatibility or a server gap - to close; do not advertise them as normal binary conversions without tests. -2. **LONG_ARRAY is not a supported array conversion.** Auto-creation explicitly - rejects it, despite public client setters and a decoding cursor. For an - existing array column, the append branch accepts either array cursor, but - the rank check runs only for DOUBLE_ARRAY and does not compare element type. - Thus the existing-target route is not proof of correct LONG_ARRAY ingestion - or cross-element conversion. Resolve the server validation/compatibility - gap rather than inventing client-side element casts to cover it. -3. **CHAR-to-text uses valid UTF-8, not the server's unsafe VARCHAR path.** The - legacy server formatter truncates a non-ASCII CHAR when targeting VARCHAR. - Iteration 2.47 deliberately fixes that behavior in schema mode by encoding - any non-surrogate CHAR as target-native UTF-8 for both STRING and VARCHAR. - A lone UTF-16 surrogate has no valid standalone UTF-8 representation and is - rejected locally instead of being replaced or corrupted. Native CHAR writes - still preserve all 16 bits. -4. **Designated timestamp fallback is not a conversion catalogue.** Its special - branch can pass a fixed-width cursor to timestamp storage without the - ordinary-column source-type allowlist. Do not derive new public casts from - bit reinterpretation. The client must implement `at`/`atNow` using the pinned - designated unit and the row-completion contract above. -5. **Malformed decimal wire metadata is outside the client contract.** SQL declarations enforce - precision 1..76 and scale 0..precision. The decimal cursor stores a signed - Java scale byte; auto-creation interprets it unsigned and passes it into type - construction whose range checks are assertions. Existing-target conversion - also trusts the unsigned scale. Non-bitmap DECIMAL64/128 sentinel limbs are - formatted as physical minimum integers, while the DECIMAL256 sentinel tuple - formats as an empty non-null string. These are server validation gaps, not - useful schema semantics. Phase 5e nevertheless accepts the physical values - reachable through public low-level decimal constructors and mutators. It - mirrors the current server conversion by promoting to Decimal256 and - interpreting the scale as an unsigned QWP byte; it does not claim that wire - scales 77..255 are valid SQL decimal metadata. Public null sentinels remain - no-ops. Rejecting malformed metadata on the server or hardening the Decimal - API is a separate production change. - -Outside these explicitly listed boundaries, unlisted pairs are not missing -features. In particular, no server conversion supports BOOLEAN-to-SYMBOL, -numeric-to-BOOLEAN, FLOAT/DOUBLE-to-DATE/timestamp, timestamp-to-numeric/DATE/ -SYMBOL, DATE-to-numeric/timestamp, or UUID/LONG256/CHAR/IPv4/geohash/decimal-to- -SYMBOL. SYMBOL does not feed scalar parsers. Neither string-to-IPv4 nor -IPv4-to-integer is accepted: `ipv4Column(text)` parses locally and INT-to-IPv4 -is a distinct migration path. Arrays do not convert to/from scalar values; -decimal and geohash inputs do not convert to ordinary numeric targets. - -### Source-null rules for text output - -Approved on 2026-09-11 for LONG-to-STRING/VARCHAR/SYMBOL only: supplied -`Long.MIN_VALUE` becomes target SQL NULL before formatting, regardless of -surrounding rows. This is an effective write, not a setter no-op; -first-value-wins still applies. Ordinary LONG values use canonical signed -decimal text. Implemented and validated in iteration 2.23 (journal D039). - -Separately approved on 2026-09-11 for UUID-to-STRING/VARCHAR: only the pair with -both limbs `Long.MIN_VALUE` becomes target SQL NULL, as an effective first write. -Every other pair, including either single minimum-valued limb, produces canonical -lowercase UUID text. Implemented and validated in iteration 2.24 (D040). -Native UUID-to-UUID bytes and UUID-to-SYMBOL rejection remain unchanged. - -Separately approved on 2026-09-11 for FLOAT/DOUBLE-to-STRING/VARCHAR/SYMBOL: -every supplied NaN becomes target SQL NULL as an effective first write, -independent of bitmap context. Validate target support and duplicates first. -Finite values, signed zero and infinities use server-equivalent text; FLOAT -widens to DOUBLE before formatting. Legacy behavior remains unchanged. -Implemented and validated in iteration 2.25 (D041). - -Real-server characterization confirms the contrast below for all eleven pairs. -D035 records LONG/UUID evidence; D041 adds the six floating-point pairs: - -| Supplied value | Legacy, no bitmap | Legacy, bitmap marks value present | Schema-mode text rule | -| --- | --- | --- | --- | -| LONG `Long.MIN_VALUE` | NULL | Text `"null"` (not SQL NULL) | NULL (approved) | -| UUID with both limbs `Long.MIN_VALUE` | NULL | `80000000-0000-0000-8000-000000000000` | NULL (approved) | -| FLOAT/DOUBLE NaN | NULL | Text `NaN` (not SQL NULL) | NULL (approved) | - -Other LONG values retain signed decimal formatting; other UUID limb pairs -retain canonical lowercase UUID formatting, including a pair with just one -minimum limb. Legacy-mode bytes and replay remain unchanged. Implementation -must document this deliberate departure from bitmap-present legacy results. -Do not extend the rule to other input families or targets by analogy. - -### Integration work is separate from conversion coverage - -The table does not include missing-table/column inference or Sender lifecycle -integration. Iteration 2.15 implements first-effective-write schema resolution, -designated timestamp row completion and schema-mode adoption (since moved from -the row boundary to the batch boundary). Iteration 2.18 -adds inference for the already-supported setter families, not the missing -families in the conversion inventory. Preserve each public overload's null/no-op -and duplicate rules: -for example decimal/array null and IPv4 text null can be no-ops, binary null -rejects, and a null `Instant` is not a missing timestamp. No fallback to sending -unconverted values, new conversion framework or send-time transformation is -implied by this inventory. - -### Source map - -Line numbers identify this audited working-tree snapshot; method names are the -durable lookup points. - -- [Client Sender][conversion-sender]: public overloads and wire choice, - lines 1118–3017. [Binding][conversion-binding]: implemented setters, - lines 108–403; target selection/parameter rejection, lines 473–545. - [Client buffer][conversion-buffer]: string/binary exception-atomic append, - lines 710–763, 1211–1230 and `restoreStringAppendPositions` (1668). -- [Server cursor selection][conversion-cursors]: `initializeColumnCursor`, - lines 469–562; [fixed-width cursor][conversion-fixed-cursor]: `advanceRow` - and `isCurrentValueSentinelNull`, lines 67–87 and 301–313. -- [Server conversion dispatch and values][conversion-dispatch]: decimal target - entrypoints (367–394), `putDecimalToDecimalColumn` (2087–2109), and decimal - rescaling/range/write helpers (2019–2085, 2433–2522). -- The same [server value-conversion source][conversion-values] implements `putIntegerToNumericColumn` - (1137), integer/float text and symbol writers (790–1041), - `putStringToNumericColumn` (1399), `putStringToTimestampColumn` (1542), - `putStringToLong256Column` (1377), `putCharColumn` (334), - `putBinaryColumn` (207), `putTimestampToStringColumn` (1851) and - `formatFixedOtherValue` (1917). -- The same [value-conversion source][conversion-values] implements decimal - text output (398–516), integer/float input (684–902), string input (1275, - 2282–2429), rescaling/range checks (2019–2279, 2433–2522), geohash input/ - output (1046–1133, 1299–1373), and array copying (`putArrayColumn`, 156). -- [Decimal cursor][conversion-decimal-cursor], lines 73–91 and 196–281; - [array cursor][conversion-array-cursor], lines 179–310; - [geohash cursor][conversion-geo-cursor], lines 122–188; - [GeoHashes][conversion-geohashes], `fromStringTruncatingNl` (238); - [decimal type construction][conversion-column-type], `getDecimalType` (320); - [SQL decimal validation][conversion-sql-parser], lines 269–297. -- [UTF-8 decoding][conversion-utf8], `utf8CharDecode` (1354–1401); - [UTF-8 sink][conversion-utf8-sink], `putAscii(char)` (128–130). - -[conversion-sender]: ../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java -[conversion-binding]: ../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java -[conversion-buffer]: ../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java -[conversion-cursors]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpTableBlockCursor.java -[conversion-fixed-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpFixedWidthColumnCursor.java -[conversion-dispatch]: ../../questdb/core/src/main/java/io/questdb/cairo/wal/WalColumnarRowAppender.java -[conversion-values]: ../../questdb/core/src/main/java/io/questdb/cairo/wal/WalColumnarRowAppender.java -[conversion-decimal-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpDecimalColumnCursor.java -[conversion-array-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpArrayColumnCursor.java -[conversion-geo-cursor]: ../../questdb/core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpGeoHashColumnCursor.java -[conversion-geohashes]: ../../questdb/core/src/main/java/io/questdb/cairo/GeoHashes.java -[conversion-column-type]: ../../questdb/core/src/main/java/io/questdb/cairo/ColumnType.java -[conversion-sql-parser]: ../../questdb/core/src/main/java/io/questdb/griffin/SqlParser.java -[conversion-utf8]: ../../questdb/core/src/main/java/io/questdb/std/str/Utf8s.java -[conversion-utf8-sink]: ../../questdb/core/src/main/java/io/questdb/std/str/Utf8StringSink.java - -## Local errors and stale schemas - -Conversions run inside the producer's row-rollback guard. A failed conversion -or lookup cancels the partial row and removes bindings introduced only by that -row; it does not discard completed rows or halt the sender. Diagnostics identify -the table, column, target type, input type and conversion failure. - -Expose these failures through a schema-specific `LineSenderException` subtype -with stable machine-readable reasons. Callers must not parse error messages to -decide recovery. The proposed reason codes and actions are: - -| Reason | Caller action | -|---|---| -| `INVALID_VALUE` | Correct or reject the input; reconstruct the whole row if writing it again. Retrying the same input against the same snapshot will fail. If server DDL has since made the input valid, the next batch after an acknowledged flush validates against the new schema. | -| `SCHEMA_UNAVAILABLE` | Retry the whole row when metadata can be obtained, for example after a timeout or temporary outage. | -| `ACCESS_DENIED` | Resolve authorization; unchanged credentials/permissions do not make an immediate retry useful. | -| `UNSUPPORTED_FEATURE` | Use a supporting endpoint/client for an extension required after upgrade or by pending extended SF, or for an unsupported target type. Do not downgrade or fall back to raw encoding. | - -An old server's absent confirmation before upgrade is normal legacy negotiation, -not `UNSUPPORTED_FEATURE`. - -For failures raised during row construction, the partial row has already been -cancelled. Retrying `flush()` cannot recover it, although a flush may still -publish earlier completed rows. Do not mark these errors as flush-retryable via -the existing `isRetryable()` convention; the schema reason determines whole-row -recovery. Existing transport/flush failures and terminal server NACKs keep their -own delivery and recovery semantics. Test the reason, row outcome and allowed -next public API operation, not only the diagnostic text. - -A conversion rejection is final for the snapshot the batch pinned. The failed -setter reports `INVALID_VALUE` or `UNSUPPORTED_FEATURE` directly; it never -performs a network lookup, so a rejection costs no round trip and cannot fail -for a transport reason. A cached schema therefore keeps rejecting an input that -DDL has since made valid until the batch boundary brings the new snapshot (see -the next section). Lookup and authorization errors surface only from the path -that actually looks metadata up: the first row of a batch on a cache miss. - -Do not retry only the failing setter after a refresh: earlier values in the -same row have already been converted under the old snapshot. The sender does -not retain original inputs for automatic whole-row reconversion. - -## One schema snapshot per table per batch - -Keep two concepts separate: - -- The latest server snapshot, learned through describes or ACK feedback. -- The encoding snapshot pinned to a table for the pending batch. - -All rows a table holds in one batch use one encoding snapshot. Its identity -describes the schema used to resolve server-backed bindings, with explicit -inferred bindings for any missing columns. It is not replaced by the latest -cache identity when the block is sent. Wire column definitions still describe -the actual bytes; the identity does not replace those definitions. - -The sender resolves the cache only when a table's buffer holds no rows: at the -first effective write of the table's first row in a batch, including -timestamp-only rows. From then on every row of that table in the batch reuses -the pinned binding without consulting the cache, so an ACK that lands mid-batch -with a newer snapshot cannot split the batch or relabel buffered values: - -``` -Server v10: count INT -> this batch: v10, INT, four-byte values -Server v11: count LONG -> next batch: v11, LONG, eight-byte values -``` - -Adoption happens at the batch boundary. After a successful `flush()` or an -explicit `reset()` the table buffer is row-less again, and the next row -compares the pinned identity with the cache; a different identity replaces the -buffer's layout (`QwpTableBuffer.reset()` keeps column definitions, so the -buffer is cleared and rebound in place) before accepting the first value. A -failed flush retains the batch and its snapshot. - -There is no explicit refresh API. The cache stays current through the server: -a frame that carries a stale identity makes the server attach the table's -current schema to the ACK, and the send loop applies that feedback before it -advances the ack watermark, so an application that flushes and waits for the -acknowledgment before starting the next batch adopts the change deterministically. -When the server cannot produce the snapshot it answers `INVALIDATE_ALL`, the -cache empties, and the next batch's first row looks the table up again and -reports `SCHEMA_UNAVAILABLE`, `ACCESS_DENIED` or `UNSUPPORTED_FEATURE` from the -setter. A reconnect also clears the cache. A producer whose every new row is -rejected by a stale snapshot never ships a frame and therefore never receives -feedback; recovery from that state is a reconnect or a new sender. An explicit -refresh was considered and dropped: a caller cannot tell a stale-snapshot -rejection from invalid input, so there is no sound rule for when to call it. - -A frame therefore never carries two generations of one table, and the sender -keeps no retired buffers, pending-generation lists or per-operation freshness -bookkeeping. Persisted frames from before this rule may still hold two blocks -of one table; the replay path reads them unchanged. - -## Schema feedback through writes - -Each table block in an extended frame carries its name and pinned -`(tableId, metadataVersion)`, or an explicit unknown identity for a missing -table. Multi-table writes carry a separate identity per block. The name remains -the write target, including after drop/recreate; the pair is not a -compare-and-swap precondition or a schema lock. - -The server processes the actual wire values using normal schema resolution and -conversion checks, then compares identities, including changes from auto-create -or added columns. It remains authoritative and may convert the already -converted values again if its schema changed. - -| Outcome | Response | -|---|---| -| Identity matches | Ordinary ACK or NACK | -| Identity differs or is unknown; write succeeds | ACK plus current identity and full schema | -| Identity differs or is unknown; write fails and metadata is available and authorized | NACK plus current identity, full schema and error | - -A version difference alone does not reject data. An old INT block can still -succeed against a new LONG column. Returned metadata must be a coherent -snapshot: identity and column definitions describe the same schema. Updates -are keyed by table name. - -ACKs are cumulative. Accumulate and coalesce pending schema updates until an -eligible response carries them; preserve them across socket backpressure, -deferred frames and commit boundaries. A metadata update never makes otherwise -unacknowledged data safe to acknowledge. - -Order describes and write-response metadata consistently per table name. A -delayed ACK must not overwrite a newer describe, including after drop/recreate. -Do not order different table IDs numerically. Ignore responses from old -connections and cancelled requests. - -For feedback, reserve the top two bits of the response status byte as a mode; -the low six bits retain the existing ACK/NACK status: - -| Mode bits | Payload after the status byte | -|---|---| -| `0x00` | Existing response body, unchanged; no feedback | -| `0x80` | Existing response body, then the complete schema update list | -| `0xC0` | Existing response body only; invalidate all latest-schema cache entries | -| `0x40` | Reserved; reject | - -Both feedback modes require successful schema negotiation. Ordinary cumulative -ACKs and NACKs may carry feedback; durable ACKs retain their existing format -and never carry mode bits. No-feedback responses remain byte-identical even on -negotiated connections. Base response statuses must fit in six bits. - -The update list is `count:u16 LE`, followed by that many entries: -`nameLength:u16 LE`, UTF-8 table name, `schemaPayloadLength:u32 LE`, schema -payload. Count must be positive; names must be valid and unique. Each payload -reuses the SCHEMA control payload without its QWP frame header: -`kind=2:u8`, `requestId=0:i64 LE`, result and any known-schema body. Zero is -reserved for write feedback; DESCRIBE replies still require a positive request -ID. Share schema payload validation, including safe skipping of unknown type -parameters. Column names must satisfy the server's column-name rules and be -unique under its case-insensitive comparison. Validate these once in the shared -schema decoder, before constructing a schema response. Validate the whole -response, exact lengths and trailing-byte exhaustion before publishing metadata -or processing its ACK/NACK outcome. - -Bound the complete `UPDATES` binary response payload to 1 MiB and its WebSocket -header plus payload to the configured response-buffer capacity. If the complete -update set cannot fit, send `INVALIDATE_ALL` instead; never truncate the list, -paginate it or send extra cumulative ACKs to drain metadata. This fallback adds -no bytes to the base ACK/NACK, including at an exact buffer-capacity boundary. -`NONE` and `INVALIDATE_ALL` retain existing base-response size limits: do not -impose the new 1 MiB metadata limit on an otherwise valid large base ACK. If a coherent, -authorized complete update set cannot be returned, invalidation also provides -a bounded fallback without exposing metadata. - -Invalidation affects only the latest-schema cache used for future row starts. -The next use describes the table normally; an individually oversized schema -can then report the existing lookup limit. Pinned snapshots, partial rows, -buffered/persisted bytes and ordinary/durable ACK semantics remain unchanged. - -On the server, retain bounded pending table identities/names rather than stale -encoded snapshots. The current WebSocket path bounds these by its configured -table-writer ceiling, plus at most one failing table name in a rejected group. -Recheck that dependency if table-writer eviction or deferred-group ownership -changes; the response-size limit alone is not a bound on queued names. -Sample coherent metadata and recheck authorization when -serializing each response, in the same serialized response path as DESCRIBE. -Keep committed-prefix ACK feedback separate from feedback for a rejected frame -or deferred group. A NACK may describe authorized tables encountered in that -rejected work; it must not absorb a pending earlier ACK's feedback. Partial -sends retain their exact bytes and feedback ownership until completion. Old -peers must never receive unnegotiated extensions, including during SF replay. - -## Cache and ownership - -Allow up to 1,000,000 latest-schema cache entries per connection, allocated on -demand. No TTL, polling, subscriptions or unsolicited DDL push is required. -Eviction makes the next row's schema resolution a lookup. - -The I/O thread publishes immutable entries in a thread-safe cache; all row, -binding and layout changes are producer-owned. Pinned snapshots may outlive -cache eviction or reconnect. Share immutable metadata where possible and -release it when no longer needed; the entry ceiling is not a total-memory -bound for schemas, converters and pending buffers. - -Use one outstanding lookup because the producer owns row construction; no -general request queue is needed. Its deadline starts at the producer call and -includes waiting for the I/O thread, connection and response. Never reuse a -request ID; correlate completion with the connection that sent it. Recoverable -transport loss retains the pending lookup, including one not yet sent. Resend -on the replacement connection with a fresh request ID within the original -deadline; repeated disconnects do not restart that deadline. The lookup can -time out while the store-and-forward loop continues reconnecting indefinitely. -Timeout, interruption, shutdown and terminal failure release waiters. Ignore -cancelled or unmatched replies entirely, including their metadata. Explicit -denial, unavailability and unsupported responses complete the lookup without -retrying it. - -Validate the fixed control header and request ID before parsing a direct -reply's schema. An identifiable stale reply is ignored even if its schema body -is malformed. Invalid headers that cannot safely identify a reply still fail -the connection. Parse outside the coordinator monitor; if parsing fails, -recheck the existing request identity and deadline before accepting the error -as a current connection failure. Successful parsing retains the final check -before publication. A malformed current reply cancels its matching lookup -atomically with that identity check before recycling the connection; malformed -headers that cannot identify a reply cancel the connection's pending lookup. -Protocol errors retain this immediate failure policy even when transport loss -would preserve the lookup for reconnection. - -Cache known schemas and confirmed missing-table results, not denial or -unavailability. A forced refresh removes the previous entry before lookup; -failure does not restore it. Independently received new feedback remains usable. -Existing entries remain usable during an outage; -a later cache miss may wait for reconnection within its own deadline. Installing -a replacement connection clears the latest cache, without changing pinned -snapshots or cancelling a newer request that has not yet been sent. - -A lazy insertion-order cache under the coordinator's monitor is sufficient for -this slice. Normalize names using the server's character comparison rules. -Apply complete validated replies in receive order: server-side metadata -sampling and serialized response sends provide the ordering guarantee above. -Do not add a client wire-sequence barrier: an ACK for older data can legitimately -contain metadata sampled after a newer DESCRIBE. - -## Delivery and remaining limits - -Flush, auto-flush and close encode or publish blocks in their pinned format -without schema lookups or value reconversion. SF persists the original encoded -bytes, frame flag and any pinned identity; replay requires neither original -inputs nor a fresh describe. Extension confirmation is required after upgrade -or when recovering extended SF, but legacy-only recovery before upgrade works -with old servers. No send-time transformation is needed in either mode. A stale -identity can produce repeated schema feedback but is not itself a reason to -reject a replay. Replaying legacy data preserves it; it does not retroactively -provide local conversion validation for those rows. - -Schema feedback is independent of ACK/NACK delivery semantics. A NACK carrying -a new schema still rejects the batch: `SCHEMA_MISMATCH` remains TERMINAL under -[NACK policy v2](qwp-nack-policy-v2.md). Metadata cannot repair persisted bytes, -authorize automatic retry after a terminal error, discard data or advance -ordinary or durable-ACK watermarks. There is no `discard()` API in this feature. - -Concurrent DDL can reject data that was locally valid. It can also make the -server apply another conversion to already converted values. Original inputs -may have lost precision or formatting during the first conversion, so replay -is not equivalent to converting those original inputs against the new schema. -The sender provides neither schema locking nor transparent reconversion after -DDL. Server checks and NACK recovery remain necessary. - -## Simplification opportunities - -These guide integration; the status section records what is implemented. -They do not change the compatibility contract. - -1. **One owner of row data.** Reuse the schema conversion logic with Sender's - existing table buffers. Do not add a parallel buffer lifecycle with separate - row counts, rollback, flush accounting or close handling. Preserve the - distinction between the latest schema and each pending block's pinned - snapshot; reusing storage must never overwrite or relabel pending rows. -2. **One owner of the connection.** Route schema lookups through the existing - I/O loop using a small, bounded request handoff. Reuse its connection, - reconnect and shutdown lifecycle instead of adding another socket owner or - a general-purpose request framework. Keep request correlation, deadlines - and cancellation explicit. The producer still waits for required metadata - before accepting a value and owns conversion; the I/O loop handles the - exchange. -3. **One schema representation.** Share immutable metadata and schema payload - encoding/validation between DESCRIBE and ACK/NACK feedback, while retaining - their distinct message framing and delivery rules. Reuse authorization and - size checks. Keep full snapshots; avoid schema patches, subscriptions or an - additional conversion-version handshake. Treat the encoded 32-bit - `ColumnType` as the canonical source of decimal precision/scale, geohash bits - and array element/rank. Keep the existing parameter bytes reserved; do not - add a second per-type parameter model until the encoded type cannot represent - a required contract. -4. **A short view of remaining work.** Keep a concise implementation-status - checklist near the beginning of the design, separate from the intended - contract. Distinguish tested components from ordinary Sender activation. - Keep decision history and detailed test evidence in the journal so readers - do not need to reconstruct current status from past iterations. Link to - that evidence rather than maintaining duplicate test logs in the design. - Retain method and file names in the source map, but remove brittle source - line numbers when the document is next compacted. -5. **Reuse inferred column definitions.** A confirmed missing target needs no - second schema registry. Let the existing buffer pin the first effective - native type and own rollback. Reuse the existing typed-rejection refresh - when a later row conflicts. Add a guard only for a reachable contract case; - `at()` already finishes a row, so another within-row designated-timestamp - policy is unnecessary. D033 applies this simplification. -6. **Keep local and server rejection tests distinct.** Use public Sender to - verify early errors and row recovery; use explicit legacy frames when the - assertion concerns server conversion or rejection. Do not make a shared - error helper accept either path: that can turn a missing server test into - an apparent pass. Await the relevant ACK when a test depends on feedback, - rather than adding sleeps or accepting multiple error reasons. -7. **Stable negative tests.** For general rejection/rollback checks, use a pair - the server actually rejects, such as LONG-to-BINARY. A temporarily missing - converter such as LONG-to-STRING makes that test obsolete when support is - added. Keep explicit pending-conversion characterization separately. D039 - applies this rule without changing the existing SQL or rollback assertions. -8. **Keep common write paths small.** Isolate an infrequent conversion when - measurements justify it, without adding a conversion framework. In D041, - moving text formatting out of the numeric switch restores native-path - inlining. Shared helper names are not proof of formatter compatibility: - compare actual output before reusing or changing legacy utilities. -9. **Port only the timestamp behavior QWP uses.** The STRING-to-timestamp - converter needs eleven fixed formats, not a general format compiler. Reuse - public JDK timezone metadata, but preserve the server's token ordering, - calendar edge cases and parsed-year recurring-rule calculation. A plain JDK - instant lookup was disproved by an extreme-year input. D043 records the - counterexample and the accepted narrow adapter. Differential and E2E tests - caught a missing fixed-fraction fallback and a retained-leap calendar quirk - before acceptance. No new protocol metadata or parser framework was needed. - Timestamp output likewise uses one fixed formatter and the existing text - sink; it does not need a general calendar-formatting framework (D044). -10. **Reuse the full setter lifecycle.** Activating a conversion includes the - existing typed-error rollback and schema-refresh handling, not just the - binding call. Test refresh through the newly activated setter itself. - D045's CHAR generation test caught a missing catch; adding the standard - catch fixed it without a new abstraction or changed cache policy. -11. **Reuse one decimal conversion tail.** Initialize the existing per-column - scratch value from each source, then share exact rescaling, precision/storage - checks and direct target-width append. LONG therefore needs no separate - decimal encoder, registry or retained state. Its source-null case only locks - the known target scale before marking the bitmap. D049 applies this rule. -12. **Keep one decimal text parser and append tail.** Sync the shared decimal - parser's precision calculation with the current server instead of creating a - schema-only fork, then route the parsed Decimal256 scratch through the same - phase-5a append tail. Enforce exact special-value matching centrally according - to the parser's documented contract, rather than adding target-width branches. - D050 applies this rule. -13. **Implement intentional conversions, not every reachable cast.** The server - audit is evidence, not a requirement to reproduce parser accidents or - validation gaps. Keep BINARY-to-parser conversions, LONG_ARRAY and malformed - decimal metadata out until their server contracts are deliberately resolved. - For CHAR-to-VARCHAR, D069 applies this rule by emitting correct UTF-8 instead - of copying the server's one-byte truncation bug. -14. **Keep the schema deadline fixed.** Initial negotiation and its first - describe share one 30-second end-to-end budget; later cache-miss describes - and forced refreshes each get their own 30-second budget. Transport retries - retain that original budget. Do not add public configuration, retries of - completed lookup errors, another socket, a request queue or separate timeout - policy per path. -15. **Prefer narrow family helpers to a conversion framework.** A shared helper - for BYTE/SHORT/INT numeric targets is justified because those setters have - the same range-check and append mechanics. Do not introduce a converter - registry or graph. Do not refactor the established LONG hot path without - measurements showing that the change is useful and safe. -16. **Make compatibility a standing gate, not a feature iteration.** Run the - real old/new client-server binary matrix in CI and before release. Pin the - last pre-extension releases and verify their checksums; do not use moving - `latest` versions. Reuse one public-API probe compiled against the old client - and launch each client in an isolated JVM. Keep exact upgraded-sender - no-send, watermark and replay assertions in the faster deterministic socket - and replay tests instead of adding a flaky process failover rig. Each numbered - conversion iteration still keeps a focused current-server E2E test green; - it does not reimplement negotiation or compatibility. -17. **Plan by conversion mechanism.** Group target paths that share a wire - representation, conversion rule and rollback behavior. Do not create one - iteration per matrix cell, and do not combine unrelated server-boundary - decisions merely because they appear in the same source audit. -18. **Treat textual decimal as an existing source type.** Parse into the - Sender-owned Decimal256 scratch value, then reuse the native decimal append - and text-formatting tails. Keep parsed specials separate only because they - mean an effective target NULL while typed decimal null sentinels are no-ops. - Normalize an unset null-only scale at the encoding boundary; do not add a - second scale state, parser or converter registry. -19. **Keep IPv4 as one primitive path.** Reuse the native four-byte column for - IPv4 targets and the existing reusable numeric sink for canonical text. - Normalize zero to the target bitmap before append. Do not add an address - object, parser fork, converter registry or per-row string allocation. -20. **Keep LONG256 as four primitive limbs.** Reuse the native 32-byte column - for identity and the existing numeric sink plus hex appenders for text. - Normalize the four-limb sentinel to the target bitmap before append. Do not - add `BigInteger`, a LONG256 object, a formatter registry, retained source - values or per-row string allocation. -21. **Keep GEOHASH as a masked long plus precision.** Reuse native GEOHASH - storage for an exact precision match and the existing text sink for the - fixed-width bit string. The schema buffer's bitmap already separates NULL - from every packed value. Do not add a geohash object, lookup table, - converter registry, retained source value or per-row string allocation. -22. **Treat integer temporal targets as representation selection.** DATE, - TIMESTAMP and TIMESTAMP_NS all retain the integer value as a raw 64-bit - count; only the target wire type and its unit differ. Reuse the existing - integer append paths and source-null rules. Do not add a temporal value - object, unit-normalization layer or shared encoder: DATE has no timestamp - encoding discriminator, while both timestamp wire types do. -23. **Reuse one integer formatter for integer text targets.** BYTE, SHORT and - INT widen losslessly to long, so use the established reusable long sink and - direct STRING/VARCHAR or SYMBOL append paths. Do not add source-width - formatters, boxed values, per-row strings or a text conversion registry. -24. **Reuse one scale-zero integer decimal tail.** BYTE, SHORT, INT and LONG - differ in input width and null sentinel, but every non-null value begins as - an exact scale-zero integer. Pass that value and its source-null decision to - the established rescale, precision-check and target-width append logic. Do - not add one encoder per source width, a decimal conversion registry or - retained decimal objects. -25. **Treat INT-to-IPv4 as sentinel-aware representation selection.** Both - types carry one 32-bit value, but their null sentinels differ. Translate - INT null and IPv4 zero to the target bitmap, then append every other bit - pattern through the existing IPv4 column. Do not add address parsing, - widening, an address object or a general numeric-to-IPv4 rule. -26. **Extend the fixed temporal parser, not the abstraction surface.** DATE - needs the server's five fixed parse attempts and millisecond arithmetic. - Reuse the timestamp parser's calendar and timezone metadata, while keeping - DATE's grammar order, raw-millisecond fallback and null sentinel explicit. - Do not add a format compiler, `Instant`/date object allocation, retained - parser state or another conversion registry. Reject the one proven - extreme-year/named-zone server crash precisely; do not turn it into a - general defensive validation layer. -27. **Treat DOUBLE-array support as rank validation, not conversion.** Read the - rank directly from the public wrapper, require the canonical strong DOUBLE - array type, and reuse the existing array buffer and encoder. For a missing - column, keep one rank integer on its inferred column definition so later - rows cannot form a mixed-rank block. Do not copy elements into a converter, - add array parameters, build a second shape model, or generalize this into - LONG-array casting. The current server checks rank but can build storage - from the cursor's element type, so accepting a non-DOUBLE target would - reproduce a server validation gap rather than a supported conversion. -28. **Reuse initial-connect policy only at the cold-start boundary.** Let - `ASYNC` create ordinary legacy rows until the first handshake, then adopt - schema mode at the next batch boundary. Keep `OFF` and - `SYNC` strict, and let a recovered schema-required backlog override the - startup exception. Do not add raw-value storage, a deferred converter, a - second availability option or a downgrade path after schema confirmation. - Extending fallback to later cache misses would turn a connection option - into a permanent validation switch and is intentionally excluded. - -Outside the explicit pre-handshake `ASYNC` exception, these opportunities do -not justify a per-setter opt-out, raw-value fallback or send-time -transformation. The intentional conversion catalogue is complete on the -unreleased development branch. Release review must still verify the standing -gates and explicitly accept or resolve the server-contract boundaries above; -they must not be hidden behind fallback behavior. - -## Iterative implementation and testing - -Establish a working real-client/real-server E2E test first, then grow the -implementation and its tests together. Do not wait for negotiation, -compatibility and all converters to be implemented before exercising the -complete client-to-server path. Reuse the existing test infrastructure. - -### Plan after iteration 2.45 - -The permanent compatibility gate is implemented. It launches released and -current artifacts as separate processes and checks these observable contracts: - -- Current client against QuestDB 10.0.1 stores legacy LONG-to-FLOAT conversion. -- Client 1.3.9 against the current server stores the same legacy result. -- Current client against the current server stores the schema-mode result. - -The discriminator is one three-row block written to a FLOAT column: `42`, -`Long.MIN_VALUE`, then a row omitting the value. Legacy mode stores the minimum -long as a float; schema mode treats it as a source null. The old client probe is -compiled against its public API and reused unchanged with each client JAR, so -the gate tests binary compatibility as well as process interoperability. The -released JARs and their SHA-256 hashes are fixed in `versions.env`; current JARs -come from the build under test when the server pins a client snapshot. For a -released client dependency, the gate uses that exact resolved artifact from the -local Maven cache. The same script runs in stable-client CI and before release -artifacts are copied. - -The existing deterministic socket and replay tests remain the authority for exact -negotiation traffic and for a confirmed sender later reaching an old endpoint: -it sends no buffered schema frame, advances no watermark and preserves exact -bytes for replay. Splitting that state transition across two server processes -would add timing and orchestration without improving the contract assertion. -The gate adds no protocol mode, adapter, send-time transformation or production -code. - -Iteration group 2.39 is complete: IPv4, LONG256 and GEOHASH each use -target-native identity or text representation, with separate public E2E, -exact-wire, rollback, rebind, old-peer and replay coverage. The group added no -new conversion framework or server production code. - -Iteration 2.40 is committed: BYTE, SHORT and INT select DATE, TIMESTAMP -and TIMESTAMP_NS target representations, and LONG selects DATE. Source values -remain raw counts in the target unit; no unit conversion or new temporal -abstraction was added. - -Iteration 2.41 is committed: BYTE, SHORT and INT select STRING, VARCHAR -and SYMBOL target representations through the existing reusable integer -formatter and direct text/symbol append paths. The real-server conversion -diagnostic improves by exactly one error because the SYMBOL aggregate is now -green; STRING and VARCHAR continue to the independent CHAR-to-text gap. - -Iteration 2.42 is committed: BYTE, SHORT and INT select every decimal -target width through the established exact integer-decimal tail. The target's -declared precision and scale control rescaling and storage; no new decimal -representation or server production code was added. - -Iteration 2.43 is committed: INT selects target-native IPv4 storage, -normalizing the INT and IPv4 null sentinels without adding parsing, allocation -or a general numeric/address conversion. - -Iteration 2.44 is committed: STRING parses through the server's fixed -DATE grammar sequence and selects target-native DATE storage. Calendar and -timezone mechanics are reused from the existing fixed timestamp parser; no -general date-format framework was added. - -Iteration 2.45 is committed: every public DOUBLE-array overload selects -the existing DOUBLE_ARRAY wire representation only when the server's canonical -element type and rank match. The N-dimensional wrapper exposes its current rank; -confirmed missing columns pin their first effective rank. Exact-wire tests cover -ranks 1, 2, 3 and 32, raw floating-point bits, empty shapes, duplicate/rollback -behavior and generation changes. Real-server tests cover public ranks 1 through -4, stored shapes, rank rejection and partial-row recovery. - -There is no remaining intentional client conversion slice after iteration -2.47. Further datatype work must start by resolving one of the explicit -server-contract boundaries: BINARY-to-parser behavior, LONG_ARRAY validation -or malformed decimal metadata. Until then, rejecting those paths is simpler -and safer than copying accidental server behavior into the client. - -Iteration 2.47 restores CHAR-to-STRING/VARCHAR compatibility. The binding -selects the actual schema target, retains the existing two-byte CHAR path, and -uses one reusable single-character sink to append target-native VARCHAR wire -data for STRING or VARCHAR. Valid non-ASCII characters are encoded as UTF-8; -lone surrogates fail locally. No protocol, sender state, replay path or server -production change is added. The complete client suite passes 3,721 tests with -seven skips. The complete server E2E class runs 138 tests: both former CHAR -regressions and the new Unicode assertion pass, leaving 2 failures and 20 errors -from the separately tracked DATE/GEOHASH and stale legacy-expectation cases. - -Iteration 2.46 implements the revision-70 availability change with the single -`ASYNC && !cursorEngine.requiresSchema()` branch above; the existing sticky -requirement is the recovery guard. Public tests now verify exact legacy -microseconds before connection and schema nanoseconds after a supporting -handshake, timeout and interruption after schema support is confirmed, and no -legacy fallback when recovery finds schema-framed backlog. Existing partial-row -upgrade and mixed-replay coverage remains unchanged. - -Acceptance is green: the complete client suite passes 3,720 tests with zero -failures or errors, the five-test real-server schema identity/replay suite -passes, and the released-binary matrix passes current-client/old-server, -old-client/current-server and current-client/current-server combinations. This -is an availability correction, not another conversion family. - -Every slice keeps the existing observable-contract rules below: public API, -exact wire, SQL result, partial-row rollback and relevant recovery behavior. -Run canonical and packaged-client gates, the server gate against its pinned -client submodule, and affected fixed-work performance checks before acceptance. -Update the journal and exact submodule pin with every accepted slice. - -### Original milestones - -The original milestones below describe how the present architecture was built; -they remain useful as constraints for later work: - -1. **E2E baseline.** Write one typed value through the public `Sender` API to - a real server and verify it through SQL. Keep this path working throughout - subsequent changes. -2. **Schema lookup and binding.** Make that write use the server-supplied - schema. Add matching-type, missing-table and lookup-failure cases alongside - the implementation. This slice includes the minimal confirmation gate needed - to use schema mode; milestone 3 covers the broader compatibility matrix. - Land discovery controls/codecs and their real-server protocol tests before - activating the high-level Sender. Until activation, it does not request the - extension and keeps existing behavior. Do not enable schema mode merely to - test negotiation while silently retaining server-side conversions. - Producer-owned conversion components may also land before Sender activation: - exercise real discovery, local conversion/rollback, existing typed QWP bytes - and real-server SQL together. This is conversion-component E2E, not activated - schema mode. Its first increment need not transmit schema identities or - change persisted framing. Introduce the extended frame flag together with - its required negotiation and recovery checks, not as a prerequisite for - testing a local converter. The first public-Sender integration test need not - wait for all conversion families. Start with a known table, implemented inputs - and explicit flush; then extend that same path through row completion, - metadata refresh, schema changes and delivery/recovery. A narrow passing test - does not establish those other paths. No public setter may silently bypass - schema conversion on a supporting connection, even during development. -3. **Negotiation and compatibility.** Add automatic legacy-server compatibility, - mandatory schema mode on supporting servers and the one-way upgrade. Cover - unchanged legacy-client behavior, pending legacy rows and SF during upgrade, - and an old endpoint after upgrade. Test real handshake and framing behavior, - not internal capability fields. -4. **Conversions, one family at a time.** Start with STRING-to-UUID, then add - the other families in separate increments. Each gets valid, invalid, - boundary and null cases, SQL verification and partial-row rollback tests. -5. **Schema evolution.** Add ACK feedback, pinned-block transitions, changes - during row construction and whole-row retry. Test the observable sequence - and resulting data for each behavior as it is introduced. -6. **Recovery scenarios, throughout development.** Extend reconnect, - legacy/extended SF replay, interrupted-delivery and process-restart coverage - whenever the corresponding behavior changes. In particular, recovery tests - start with the first persisted-framing change, not after the other milestones. - -These are milestones, not equally sized iterations or isolated layers. Split -them into small, independently reviewable changes, each production-quality for -its implemented scope. Every iteration includes extensive tests for its new -behavior and failure paths, runs the relevant existing regression suites, and -ends with the real E2E path passing. There is no later "add E2E" or "harden it" -phase. State unfinished scope explicitly; do not release the new client -or advertise the complete extension while its contract is incomplete. - -Tests assert observable contracts: public API results and exceptions, wire -messages, SQL data and documented delivery behavior. No reflection, private -state inspection, test-only production hooks or visibility changes solely for -tests. Unit tests of genuine production component contracts are appropriate; -implementation-specific call counts and internal object layouts are not. - -Use complementary public-Sender tests for integration: a real QuestDB fixture -checks local errors and exact stored rows; a scripted socket peer checks the -negotiated requests, target wire types and pinned identities. SQL alone cannot -prove local conversion because the legacy server can perform the same cast. -A peer omitting schema confirmation proves protocol fallback, not compatibility -with an actual older QuestDB distribution; label that distinction explicitly. - -Use real servers for conversion and ingestion semantics. Scripted protocol -peers and TCP relays may inject malformed responses, hold ACKs, fragment traffic -or disconnect at controlled points; they do not replace real-server E2E tests. -Coordinate concurrent tests through observable events with bounded waits, not -arbitrary sleeps. Seeded fuzzing supplements deterministic cases and must leave -reproducible failures with an independent expected-data model. - -A representative conversion test buffers valid row A, starts row B and supplies -an invalid UUID string to `stringColumn()`, then writes valid row C and flushes. -Assert that the setter throws and SQL contains A and C, with no part of B. -Observe encoded UUID values on the wire where needed to distinguish local -conversion from server-side parsing. Extend the scenario with SF and controlled -recovery as those paths change; do not substitute an assertion that an internal -rollback method ran. - -Process-restart coverage must also create its backlog through public Sender -setters, not only by appending constructed frames to the replay engine. Wait -for local publication and captured data, prove zero server delivery, then force -the producer JVM to stop without closing Sender. A fresh Sender using the same -directory and identity must replay identical QWP payloads and produce exact -rows, including when those payloads contain different pinned schema versions. -Reopening after ACK must not replay acknowledged rows. This tests process death, -not power loss; zero initial delivery is what permits exact row-count assertions -without claiming general exactly-once delivery. - -## Acceptance checks - -- New clients always request the extension. A successful old-server handshake - without confirmation selects existing inferred-type behavior, with unchanged - wire framing, server conversions, missing-value semantics and SF delivery. - Supporting servers enable schema mode automatically; no option disables it - after confirmation. -- With no completed handshake and no recovered schema-required backlog, - `ASYNC` accepts flag-clear legacy rows without waiting. Verify offline startup - followed by both old and supporting servers, handshake before the first row, - handshake during a partial row, and a later old-to-new upgrade. Pin legacy - conversion results where they differ from schema mode, including timestamp - precision, overflow and source-null handling. -- Failed handshakes, timeouts and invalid confirmation are not classified as - old servers. `OFF` and `SYNC` remain strict. Recovered schema-framed backlog - overrides `ASYNC` fallback and sends no data until schema support is confirmed. -- Existing clients against new servers see byte-compatible QWP v1 behavior, - with no unsolicited schema messages or extended responses. The base protocol - version remains 1 for both legacy and extended traffic. -- The persisted flag selects table-header parsing. Exercise mixed legacy and - extended SF replay on a negotiated connection, and reject flagged frames on - unnegotiated connections before processing data. Restart with a mixed backlog - and prove its original bytes, order and schema requirement are preserved. -- Upgrade from an old to a supporting server finishes the pending legacy batch - unchanged and adopts schema mode on the first row after it flushes, including - on a table that already flushed legacy rows. Published legacy frames and SF - remain unchanged and ordered; new batches use the extended layout. No - send-time conversion occurs. -- Reconnect and background replay re-negotiate support. After upgrade, an old - endpoint receives no pending data and causes neither downgrade nor watermark - advancement. The requirement survives reconnect even with a legacy-only - backlog; recovery detects pending extended SF before sending a legacy prefix. - Legacy-only recovery by a fresh sender remains compatible with old servers. -- A bad pre-handshake `ASYNC` legacy row is rejected through the existing - server-error and terminal-SF path; it is never silently repaired, converted - or acknowledged. Document the resulting queue-stop and recovery behavior. -- The downgrade procedure drains every pending extended slot with a supporting - client under the configured ACK policy. Interrupted or timed-out draining - leaves recoverable data and does not qualify as safe to downgrade. -- Write-only credentials can discover the schema and ingest without SELECT - access. Denied/revoked credentials receive a classified failure and no - unauthorized schema in either DESCRIBE or write-response feedback. -- In schema mode, first use waits for schema. Valid UUID strings and typed UUID - inputs produce the same target representation; malformed strings fail before - row commit/SF. -- Shared conversion vectors cover every input/target pair in the contract, edge - values, nulls, text formatting, timestamp units, decimal rescaling, array - ranks on later rows and geohash precision. Verify final server values too. -- Different input setters can feed one known target column. Inferred missing - columns, duplicate/no-op and name behavior remain covered. -- In schema mode, omitted values and explicit nulls follow target missing-value - semantics, independent of earlier input setter types. Cover boolean-to-STRING omission - becoming NULL, explicit false remaining "false", non-nullable targets and - unchanged results for legacy clients and new clients in legacy mode. -- For LONG input to each numeric target, test `Long.MIN_VALUE` with and without - another omitted value in the same block. Schema mode always emits target - missing/null; legacy mode retains its existing context-dependent behavior. - Duplicate suppression and unsupported-pair rejection still precede normalization. -- For LONG-to-STRING/VARCHAR/SYMBOL, verify canonical decimal bytes, MIN target - NULL versus omission, and MIN-first duplicate suppression. Preserve independent - legacy bitmap-present MIN-to-literal-"null" tests. Distinct/repeated symbols - must retain their own dictionary values when the formatter sink is reused. -- Timestamp overloads test every supported unit's range boundaries, negative - fractions and `Instant` precision at both target resolutions. Include valid - negative instants whose intermediate seconds multiplication would overflow, - duplicate suppression, local failure/rollback and unchanged legacy behavior. -- A stale rejection is local: it issues no describe, cancels the whole row and - leaves the batch pinned. No setter splices new rules into a partial row. -- Local errors expose the expected reason and recovery action. An invalid row - stays cancelled after flush; whole-row retries in the next batch after ACK - feedback or a recovered lookup can succeed without losing earlier completed - rows. Access denial and unsupported functionality are not mislabeled as - transient lookup failures or flush-retryable errors, and an `INVALIDATE_ALL` - ACK makes the next batch's lookup report them from the setter. -- ACK metadata arriving mid-batch leaves that batch unchanged. The next batch - adopts the snapshot with matching wire types and identity in its own frame; - exercise INT-to-LONG and cached designated timestamps. -- `reset()` discards rows without shipping a frame, so it delivers no feedback; - the next batch pins the cached snapshot. Transitions preserve pending - accounting, frame order, transaction boundaries, backpressure and completed - rows on error. -- Auto-create, multi-table writes, cumulative ACKs, deferred commits, delayed - sends and describe/ACK interleaving preserve schema feedback and watermarks. -- After schema confirmation, lookup timeout, offline cache misses, eviction, - reconnect and shutdown neither bypass required resolution nor invalidate - pinned data or strand callers. `ASYNC` does not enable later fallback. -- Unsupported target types fail explicitly when used; unrelated columns with - unknown types do not block supported writes. The million-entry cache grows - on demand and obsolete pinned metadata is released. -- Flush, close, byte-identical SF replay and terminal-error handling retain their - data semantics. Concurrent DDL can still NACK; metadata never repairs or - silently acknowledges rejected data. From ccef6c1e46d82f3e414db99acddb3d589a944f1f Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 18 Sep 2026 16:50:47 +0200 Subject: [PATCH 31/51] Reuse existing QWP state for eager table schema lookup --- .../main/java/io/questdb/client/Sender.java | 8 +- .../qwp/client/QwpWebSocketSender.java | 201 ++++++++-------- .../io/questdb/client/impl/PooledSender.java | 7 +- .../QwpSchemaSenderIntegrationTest.java | 220 ++++-------------- .../qwp/client/QwpWebSocketSenderTest.java | 75 ++---- .../client/test/impl/SenderPoolTest.java | 49 ---- 6 files changed, 158 insertions(+), 402 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index a736052d4..6810f23c1 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -758,15 +758,15 @@ default Sender shortColumn(CharSequence name, short value) { Sender symbol(CharSequence name, CharSequence value); /** - * Select the table and start a new row. Call this before every row, including consecutive rows for the same table. - * It's an error to add a column or complete a row without calling this method first. + * Select the table for a new row. This is always the first method to start an error. It's an error to call other + * methods without calling this method first. *
      * After calling this method you can start adding columns to the row and then call {@link #atNow()} or {@link #at(Instant)} * to finalize the row. You can then start a new row by calling this method again. *
      * If you want to cancel the current row, you can call {@link #cancelRow()}. - * QWP WebSocket senders also resolve and pin the row's schema contract here, so this call may wait for or fail - * schema discovery. Column setters do not initiate schema discovery. + * QWP WebSocket senders resolve the table's schema here when schema mode is active, so this call may wait for or + * fail schema discovery. Setters may resolve it again when an implicitly started row crosses a batch boundary. * * @param table name of the table * @return this instance for method chaining diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bf332d6df..1fd783dd8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -252,12 +252,6 @@ public class QwpWebSocketSender implements Sender { private int currentBatchMaxSymbolId = -1; private volatile int currentEndpointIdx = -1; private QwpTableBuffer currentTableBuffer; - // Authoritative row lifecycle state. A row is open immediately after a - // successful table(), even before any value has been appended. - private boolean currentRowOpen; - // Contract selected by table() for the open row. Null means an explicitly - // prepared legacy row; it never permits lazy negotiation in a setter. - private QwpSchemaBinding currentRowBinding; // Tracks currentTableBuffer.getBufferedBytes() at the last point pendingBytes // was made consistent (end of sendRow(), or right after a table switch). // sendRow() advances pendingBytes by (now - snapshot) and re-snaps, which @@ -1034,7 +1028,7 @@ public void at(long timestamp, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.designatedTimestamp(timestamp, unit); sendRow(); @@ -1056,7 +1050,7 @@ public void at(Instant timestamp) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.designatedTimestamp(timestamp); sendRow(); @@ -1074,6 +1068,7 @@ public void atNow() { checkNotClosed(); checkTableSelected(); try { + bindingForEffectiveWrite(); // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { @@ -1144,7 +1139,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.binaryColumn(columnName, value); return this; @@ -1174,7 +1169,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.binaryColumn(columnName, slice); return this; @@ -1199,7 +1194,7 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.binaryColumn(columnName, ptr, len); return this; @@ -1219,7 +1214,7 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.boolColumn(columnName, value); return this; @@ -1250,7 +1245,7 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.byteColumn(columnName, value); return this; @@ -1268,7 +1263,10 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { @Override public void cancelRow() { checkNotClosed(); - rollbackRow(); + if (currentTableBuffer != null) { + currentTableBuffer.cancelCurrentRow(); + currentTableBuffer.rollbackUncommittedColumns(); + } } /** @@ -1284,7 +1282,7 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.charColumn(columnName, value); return this; @@ -1394,10 +1392,6 @@ private void close0(boolean[] restoreInterrupt) { ? cursorSendLoop.getSynchronouslySurfacedError() : null; try { - // Closing never turns an unfinished row into an implicit write. - // Keep cancellation inside the protected cleanup path so even an - // unexpected rollback failure cannot skip transport teardown. - rollbackRow(); // Only drain when both the engine and the I/O loop are wired // up — close() is also called from createForTesting() teardown // and from connect() rollback paths where one or both may be null. @@ -1625,10 +1619,10 @@ public void setSlotLockReleasedForTesting(boolean isReleased) { @Override public Sender decimalColumn(CharSequence name, Decimal64 value) { checkNotClosed(); - checkTableSelected(); if (value == null || value.isNull()) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.decimalColumn(name, value); return this; @@ -1646,10 +1640,10 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { @Override public Sender decimalColumn(CharSequence name, Decimal128 value) { checkNotClosed(); - checkTableSelected(); if (value == null || value.isNull()) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.decimalColumn(name, value); return this; @@ -1667,10 +1661,10 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { @Override public Sender decimalColumn(CharSequence name, Decimal256 value) { checkNotClosed(); - checkTableSelected(); if (value == null || value.isNull()) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.decimalColumn(name, value); return this; @@ -1688,10 +1682,10 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { @Override public Sender decimalColumn(CharSequence name, CharSequence value) { checkNotClosed(); - checkTableSelected(); if (value == null || value.length() == 0) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.decimalColumn(name, value, currentDecimal256); return this; @@ -1710,10 +1704,10 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { @Override public Sender doubleArray(@NotNull CharSequence name, double[] values) { checkNotClosed(); - checkTableSelected(); if (values == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.doubleArray(name, values); return this; @@ -1731,10 +1725,10 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { @Override public Sender doubleArray(@NotNull CharSequence name, double[][] values) { checkNotClosed(); - checkTableSelected(); if (values == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.doubleArray(name, values); return this; @@ -1752,10 +1746,10 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { @Override public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { checkNotClosed(); - checkTableSelected(); if (values == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.doubleArray(name, values); return this; @@ -1773,10 +1767,10 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { @Override public Sender doubleArray(CharSequence name, DoubleArray array) { checkNotClosed(); - checkTableSelected(); if (array == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.doubleArray(name, array); return this; @@ -1796,7 +1790,7 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.doubleColumn(columnName, value); return this; @@ -1822,7 +1816,7 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.floatColumn(columnName, value); return this; @@ -1968,7 +1962,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.geoHashColumn(columnName, bits, precisionBits); return this; @@ -2006,7 +2000,7 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence va checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.geoHashColumn(columnName, value); return this; @@ -2394,7 +2388,7 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.intColumn(columnName, value); return this; @@ -2427,7 +2421,7 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.ipv4Column(columnName, address); return this; @@ -2472,12 +2466,12 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { @Override public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence address) { checkNotClosed(); - checkTableSelected(); if (address == null) { return this; } + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.ipv4Column(columnName, address); return this; @@ -2521,7 +2515,7 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.long256Column(columnName, l0, l1, l2, l3); return this; @@ -2539,10 +2533,10 @@ public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l @Override public Sender longArray(@NotNull CharSequence name, long[] values) { checkNotClosed(); - checkTableSelected(); if (values == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2561,10 +2555,10 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { @Override public Sender longArray(@NotNull CharSequence name, long[][] values) { checkNotClosed(); - checkTableSelected(); if (values == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2582,10 +2576,10 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { @Override public Sender longArray(@NotNull CharSequence name, long[][][] values) { checkNotClosed(); - checkTableSelected(); if (values == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2603,10 +2597,10 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { @Override public Sender longArray(@NotNull CharSequence name, LongArray array) { checkNotClosed(); - checkTableSelected(); if (array == null) return this; + checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.unsupportedColumn(name, "LONG_ARRAY"); return this; @@ -2626,7 +2620,7 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.longColumn(columnName, value); return this; @@ -2709,7 +2703,6 @@ public void setCloseStartedHook(Runnable hook) { @Override public void reset() { checkNotClosed(); - rollbackRow(); // Reset ALL table buffers, not just the current one ObjList keys = tableBuffers.keys(); for (int i = 0, n = keys.size(); i < n; i++) { @@ -2735,7 +2728,6 @@ public void reset() { currentTableBuffer = null; currentTableBufferSnapshotBytes = 0; currentTableName = null; - currentRowBinding = null; cachedTimestampColumn = null; cachedTimestampNanosColumn = null; } @@ -2941,7 +2933,7 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.shortColumn(columnName, value); return this; @@ -3061,7 +3053,7 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.stringColumn(columnName, value); return this; @@ -3081,7 +3073,7 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.symbol(columnName, value); return this; @@ -3099,39 +3091,34 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { @Override public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); - if (currentRowOpen) { - throw new LineSenderException("cannot start a new row while a row is in progress" - + " [currentTable=").put(currentTableName).put(']'); - } - validateTableName(tableName); - ensureConnected(); - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - currentTableBuffer = tableBuffers.get(tableName); - if (currentTableBuffer != null) { - currentTableName = currentTableBuffer.getTableName(); - } else { - currentTableName = tableName.toString(); - currentTableBuffer = new QwpTableBuffer(currentTableName, this); - tableBuffers.put(currentTableName, currentTableBuffer); - } - // Re-snap so sendRow()'s delta math is anchored to this table's - // current byte count. The prior current table's bytes already match - // its last-snapped value (the in-progress-row guard above ensures - // no column setters ran on it since the last consistency point). - currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); - try { - currentRowBinding = prepareRowBinding(); - currentRowOpen = true; - } catch (RuntimeException | Error e) { - currentRowBinding = null; - currentTableBuffer = null; - currentTableName = null; - currentTableBufferSnapshotBytes = 0; + if (currentTableName == null || !Chars.equals(tableName, currentTableName)) { + // Prevent switching tables while a row is in progress. + if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { + throw new LineSenderException("cannot switch tables while row is in progress" + + " [currentTable=").put(currentTableName).put(']'); + } + validateTableName(tableName); + // Table changed - invalidate cached column references. cachedTimestampColumn = null; cachedTimestampNanosColumn = null; - throw e; - } + currentTableBuffer = tableBuffers.get(tableName); + if (currentTableBuffer != null) { + currentTableName = currentTableBuffer.getTableName(); + } else { + currentTableName = tableName.toString(); + currentTableBuffer = new QwpTableBuffer(currentTableName, this); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Re-snap so sendRow()'s delta math is anchored to this table's + // current byte count. The prior current table's bytes already match + // its last-snapped value (the in-progress-row guard above ensures + // no column setters ran on it since the last consistency point). + currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); + } + // Resolve schema at the explicit table boundary. Setters retain the + // same lookup as a fallback because the selected table also starts + // implicit rows after completion, cancellation, and batch flushes. + bindingForEffectiveWrite(); return this; } @@ -3140,7 +3127,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.timestampColumn(columnName, value, unit); return this; @@ -3168,7 +3155,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.timestampColumn(columnName, value); return this; @@ -3196,7 +3183,7 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) checkNotClosed(); checkTableSelected(); try { - QwpSchemaBinding binding = currentRowBinding; + QwpSchemaBinding binding = bindingForEffectiveWrite(); if (binding != null) { binding.uuidColumn(columnName, lo, hi); return this; @@ -3892,7 +3879,7 @@ private void checkNotClosed() { } private void checkTableSelected() { - if (!currentRowOpen) { + if (currentTableBuffer == null) { throw new LineSenderException("table() must be called before adding columns"); } } @@ -4373,7 +4360,8 @@ private void ensureConnected() { } /** - * Selects the wire contract while table() prepares a row. In asynchronous initial-connect mode, + * Selects the wire contract at an explicit table boundary or before the first + * effective value of an implicit row. In asynchronous initial-connect mode, * fresh senders use the legacy contract until schema support is confirmed; * recovered schema-framed data and senders that have already confirmed support * retain the strict schema path. @@ -4383,8 +4371,12 @@ private void ensureConnected() { * the cache; the first row after a flush or reset adopts whatever the cache holds * by then, which ACK schema feedback and reconnects keep current. */ - private QwpSchemaBinding prepareRowBinding() { + private QwpSchemaBinding bindingForEffectiveWrite() { QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); + if (currentTableBuffer.hasInProgressRow()) { + return pinned; + } + ensureConnected(); if (pinned != null && currentTableBuffer.getRowCount() > 0) { return pinned; } @@ -4446,7 +4438,7 @@ private static long remainingSchemaMillis(long deadlineNanos) { } private void ensureNoInProgressRow() { - if (currentRowOpen) { + if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { throw new LineSenderException( "Cannot flush while row is in progress. " + "Use sender.at(), sender.atNow(), or sender.cancelRow() first." @@ -5427,18 +5419,12 @@ private int symbolDeltaBaseline() { } private void rollbackRow() { - if (currentRowOpen) { + if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); currentTableBuffer.rollbackUncommittedColumns(); - endRow(); } } - private void endRow() { - currentRowOpen = false; - currentRowBinding = null; - } - /** * Seals the current buffer and swaps to the other buffer. * Enqueues the sealed buffer for async sending. @@ -5512,6 +5498,8 @@ private void sealAndSwapBuffer() { * Rows buffer until flush (explicit or auto-flush). */ private void sendRow() { + ensureConnected(); + // Hard guard: a single row whose bytes exceed the server's wire cap // would flush as an oversize WS frame the server closes with // ws-close[1009]. nextRow() measures the row -- padding included, @@ -5531,7 +5519,7 @@ private void sendRow() { if (pendingRowCount == 0) { firstPendingRowTimeNanos = System.nanoTime(); // The batch's first row selects its wire contract; see - // prepareRowBinding. + // bindingForEffectiveWrite. isPendingBatchLegacy = currentTableBuffer.getSchemaBinding() == null; } pendingRowCount++; @@ -5545,11 +5533,6 @@ private void sendRow() { pendingBytes += bufferedNow - currentTableBufferSnapshotBytes; currentTableBufferSnapshotBytes = bufferedNow; - // The row belongs to the completed batch from this point onward. End - // the open-row state before auto-flush so a publication failure cannot - // roll the committed row back or invite the caller to append it twice. - endRow(); - if (shouldAutoFlush()) { flushPendingRows(transactional); } diff --git a/core/src/main/java/io/questdb/client/impl/PooledSender.java b/core/src/main/java/io/questdb/client/impl/PooledSender.java index f431cb186..7b4e5f802 100644 --- a/core/src/main/java/io/questdb/client/impl/PooledSender.java +++ b/core/src/main/java/io/questdb/client/impl/PooledSender.java @@ -25,7 +25,6 @@ package io.questdb.client.impl; import io.questdb.client.Sender; -import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.line.array.DoubleArray; import io.questdb.client.cutlass.line.array.LongArray; import io.questdb.client.std.Decimal128; @@ -167,11 +166,7 @@ public void close() { // above. A stale duplicate throws here, and the finally routes it // to discardBroken, which drops it on the same stale-generation // check under the pool lock. - Sender delegate = slot.live(generation); - if (delegate instanceof QwpWebSocketSender) { - delegate.cancelRow(); - } - delegate.flush(); + slot.live(generation).flush(); flushed = true; } finally { if (flushed) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 552507712..e9c6900cb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -15,6 +15,7 @@ import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -65,7 +66,6 @@ public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exc .decimalColumn("value", "not-a-decimal") .atNow(); - sender.table("events"); sender.uuidColumn("failed_b", 11, 12); try { sender.decimalColumn("value", "not-a-decimal"); @@ -74,7 +74,6 @@ public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exc Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.decimalColumn("value", "2.5").atNow(); sender.flush(); @@ -93,11 +92,11 @@ public void testDecimalTextOverloadFormatsTextAndPreservesNullNoOps() throws Exc LongTextSchemaHandler handler = new LongTextSchemaHandler(941 + targetType, 951, targetType); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").decimalColumn("value", "00123.4500").atNow(); - sender.table("events").decimalColumn("value", "1.2300m").atNow(); - sender.table("events").decimalColumn("value", "Infinity") + sender.decimalColumn("value", "1.2300m").atNow(); + sender.decimalColumn("value", "Infinity") .decimalColumn("value", "not-a-decimal") .atNow(); - sender.table("events").decimalColumn("value", (CharSequence) null) + sender.decimalColumn("value", (CharSequence) null) .decimalColumn("value", "") .atNow(); sender.flush(); @@ -122,7 +121,6 @@ public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() thr try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) -1).atNow(); - sender.table("events"); sender.byteColumn("value", (byte) 99); try { switch (i % 3) { @@ -141,10 +139,9 @@ public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() thr Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.shortColumn("value", (short) 2).atNow(); - sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); - sender.table("events").intColumn("value", 3).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).smallIntegerNumericTable( @@ -197,7 +194,6 @@ public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exc try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) -1).atNow(); - sender.table("events"); sender.shortColumn("value", (short) 99); try { sender.intColumn("failed_b", 1); @@ -206,11 +202,10 @@ public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exc Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.shortColumn("value", (short) 2).atNow(); - sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); - sender.table("events").intColumn("value", 3).atNow(); - sender.table("events").longColumn("value", Long.MAX_VALUE).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.longColumn("value", Long.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).integerTemporalTable( @@ -263,7 +258,6 @@ public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Ex try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) -128).atNow(); - sender.table("events"); sender.shortColumn("value", (short) 99); try { sender.intColumn("failed_b", 1); @@ -272,10 +266,9 @@ public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Ex Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.shortColumn("value", Short.MAX_VALUE).atNow(); - sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); - sender.table("events").intColumn("value", Integer.MAX_VALUE).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).smallIntegerTextTable( @@ -327,7 +320,6 @@ public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Excep try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").byteColumn("value", (byte) 42).atNow(); - sender.table("events"); sender.shortColumn("value", (short) 99); try { sender.intColumn("failed_b", 1); @@ -336,10 +328,9 @@ public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Excep Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.shortColumn("value", (short) -7).atNow(); - sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); - sender.table("events").intColumn("value", Integer.MAX_VALUE).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).smallIntegerDecimalTable( @@ -388,9 +379,8 @@ public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { LongTextSchemaHandler handler = new LongTextSchemaHandler(1121, 1131, ColumnType.IPv4); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); - sender.table("events").intColumn("value", 0).atNow(); + sender.intColumn("value", 0).atNow(); - sender.table("events"); sender.intColumn("value", 0x0a000001); try { sender.intColumn("failed_b", 1); @@ -399,9 +389,8 @@ public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.intColumn("value", 1).atNow(); - sender.table("events").intColumn("value", -1).atNow(); + sender.intColumn("value", -1).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).intToIpv4Table("events", 1121, 1131); @@ -470,7 +459,6 @@ public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversR .binaryColumn("value", new byte[]{1}) .atNow(); - sender.table("events"); sender.decimalColumn("value", Decimal256.MAX_VALUE); try { sender.decimalColumn("failed_b", new Decimal64(1, 0)); @@ -479,16 +467,15 @@ public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversR Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.decimalColumn("value", new Decimal64(-125, 3)).atNow(); sender.table("events").atNow(); - sender.table("events").decimalColumn("value", new Decimal256( + sender.decimalColumn("value", new Decimal256( 0x161bcca7119915b5L, 0x0764b4abe8652979L, 0x7775a5f171950fffL, 0xffffffffffffffffL, 76)).atNow(); - sender.table("events").decimalColumn("value", new Decimal64(0, 0)).atNow(); + sender.decimalColumn("value", new Decimal64(0, 0)).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).decimalTextLifecycleTable("events", 781, 791); @@ -506,8 +493,8 @@ public void testNativeDecimalTextMatchesServerScalarFormattingForRawPublicValues sender.table("events") .decimalColumn("value", new Decimal128(Long.MAX_VALUE, -1L)) .atNow(); - sender.table("events").decimalColumn("value", wrappedScale).atNow(); - sender.table("events").decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); + sender.decimalColumn("value", wrappedScale).atNow(); + sender.decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).decimalTextTable( @@ -529,14 +516,13 @@ public void testNativeDecimalRollbackAndTargetScaleWithoutTableReselection() thr LongTextSchemaHandler handler = new LongTextSchemaHandler(601, 611, target); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").decimalColumn("value", new Decimal64(150, 2)).atNow(); - sender.table("events").uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); try { sender.decimalColumn("value", Decimal128.fromLong(100_000_000_000_000_000L, 0)); Assert.fail("expected target precision overflow"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.decimalColumn("value", new Decimal64(25, 2)).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).decimal64Table("events", 601, 611, 4, 15_000, 2_500); @@ -737,14 +723,13 @@ public void testStringToGeoHashNullWidthsRollbackAndRecovery() throws Exception 551, 561, ColumnType.getGeoHashTypeWithBits(8)); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").stringColumn("value", "zz").atNow(); - sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); try { sender.stringColumn("value", "a"); Assert.fail("expected invalid GeoHash text"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.stringColumn("value", "04").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).geoHashTable("events", 551, 561, 8, 0xff, 1); @@ -788,14 +773,13 @@ public void testStringToLong256RollbackAndRecoveryWithoutTableReselection() thro try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").stringColumn("value", null); sender.binaryColumn("value", new byte[]{1}).atNow(); - sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); try { sender.stringColumn("value", "0x0"); Assert.fail("expected odd-length LONG256 rejection"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.stringColumn("value", "0x0123456789abcdeffedcba9876543210").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).long256Table("events", 431, 441); @@ -839,14 +823,13 @@ public void testStringToCharRollbackAndRecoveryWithoutTableReselection() throws LongTextSchemaHandler handler = new LongTextSchemaHandler(391, 401, ColumnType.CHAR); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").stringColumn("value", "A").atNow(); - sender.table("events").uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); try { sender.binaryColumn("value", new byte[]{1}); Assert.fail("expected BINARY to CHAR rejection"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.stringColumn("value", "\ud83d").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).charTable("events", 391, 401, 'A', '?'); @@ -889,14 +872,13 @@ public void testTimestampToTextTargetsRollbackAndRecoverWithoutTableReselection( LongTextSchemaHandler handler = new LongTextSchemaHandler(351 + target, 361, target); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.MICROS).atNow(); - sender.table("events").uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); try { sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); Assert.fail("expected timestamp range error"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.timestampColumn("value", 1, ChronoUnit.MILLIS).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).timestampTextTable( @@ -952,7 +934,6 @@ public void testStringToTimestampTargetsUsePinnedWireAndRollback() throws Except Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.stringColumn("value", "1969-12-31T23:59:59.999999Z").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()).longTimestampTable( @@ -983,7 +964,6 @@ public void testStringToDateUsesExactTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.stringColumn("value", "1969-12-31").atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()) @@ -1039,7 +1019,6 @@ public void testDoubleArrayUsesExactTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()) @@ -1113,7 +1092,6 @@ public void testLongArrayRejectionRollsBackWithoutRefresh() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); sender.flush(); // Only the two completed DOUBLE[] rows reach the wire; failed_b is rolled back. @@ -1150,7 +1128,6 @@ public void testLongArrayRejectionKeepsPinnedSnapshotAfterSchemaChange() throws // A supported input of the new rank is still rejected against the // pinned snapshot; the batch does not discover the rank change. try { - sender.table("events"); sender.doubleArray("value", new double[]{3.0, 4.0}); Assert.fail("expected rejection against the pinned snapshot"); } catch (LineSenderSchemaException e) { @@ -1237,7 +1214,6 @@ public void testLongToTimestampTargetsUseRawUnitsAndRollback() throws Exception } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.longColumn("value", Long.MAX_VALUE).atNow(); sender.flush(); new FrameReader(handler.awaitDataFrame()) @@ -1327,7 +1303,6 @@ public void testFloatingToTextTargetsUsePinnedWireAndRollback() throws Exception Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(1)); else sender.doubleColumn("value", Double.longBitsToDouble(1)); sender.atNow(); @@ -1398,7 +1373,6 @@ public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L).atNow(); sender.flush(); @@ -1438,7 +1412,6 @@ public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.uuidColumn("failed_b", 1, 2); try { sender.ipv4Column("value", "not-an-ip"); @@ -1447,7 +1420,6 @@ public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.ipv4Column("value", ".....255.1.2.3......").atNow(); sender.flush(); @@ -1512,7 +1484,6 @@ public void testGeoHashTargetsUsePinnedTargetWireAndRollback() throws Exception Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.geoHashColumn("value", 0x12345, 20).atNow(); sender.flush(); @@ -1579,7 +1550,6 @@ public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.long256Column("value", 1, 2, 3, 4).atNow(); sender.flush(); @@ -1644,8 +1614,7 @@ public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Excepti Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - // The failed middle row is auto-cancelled; start a fresh row. - sender.table("events"); + // The failed middle row is auto-cancelled; continue without selecting the table again. sender.longColumn("value", Long.MAX_VALUE).atNow(); sender.flush(); @@ -1705,7 +1674,6 @@ public void testAsymmetricUuidRowsArePinnedAndInvalidMiddleRowRollsBack() throws } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); } - sender.table("events"); sender.uuidColumn("id", 0x2122232425262728L, 0x3132333435363738L).atNow(); sender.flush(); byte[] frame = handler.awaitDataFrame(); @@ -1786,7 +1754,6 @@ public void testKnownAdoptionWaitsForBatchBoundary() throws Exception { // The pending batch keeps its inferred snapshot: a conflicting input // fails against the inferred LONG and the failed row rolls back whole. try { - sender.table("events"); sender.stringColumn("failed_b", "partial"); sender.stringColumn("id", "11121314-1516-1718-0102-030405060708"); Assert.fail("expected inferred type conflict"); @@ -1814,13 +1781,11 @@ public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throw Sender sender = sender(server)) { sender.table("events").longColumn("id", 1).atNow(); try { - sender.table("events"); sender.stringColumn("id", "different inferred type"); Assert.fail("expected inferred type conflict"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.longColumn("id", 3).atNow(); sender.flush(); FrameReader reader = new FrameReader(handler.awaitDataFrame()); @@ -1858,14 +1823,12 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr // The pending batch never consults the server: the conflict is // local and only the partial row rolls back. try { - sender.table("events"); sender.stringColumn("failed_only", "partial"); sender.stringColumn("id", "different inferred type"); Assert.fail("expected inferred type conflict"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); } - sender.table("events"); sender.longColumn("id", 3).atNow(); long fsn = sender.flushAndGetSequence(); Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); @@ -1878,8 +1841,8 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr Assert.assertEquals(1, handler.describeRequests.get()); // The server could not describe the table for the stale frame, so - // its ACK invalidated the cache. The next batch looks the table up - // again and reports the lookup failure from table(). + // its ACK invalidated the cache. The next explicit table selection + // looks the table up and reports the failure there. try { sender.table("events"); Assert.fail("expected lookup failure"); @@ -1902,7 +1865,7 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr } @Test - public void testNullNoopUsesContractPreparedByTable() throws Exception { + public void testTableNegotiatesBeforeNullNoops() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); try (TestWebSocketServer server = schemaServer(handler); @@ -2035,27 +1998,6 @@ public void testOldPeerUsesLegacyGeoHashWireAndNeverDescribes() throws Exception }); } - @Test - public void testTablePinsSchemaBeforeFirstValue() throws Exception { - assertMemoryLeak(() -> { - SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 51, 52); - try (TestWebSocketServer server = schemaServer(handler); - Sender sender = sender(server)) { - sender.table("events"); - Assert.assertEquals(1, handler.describeRequests.get()); - - // A schema change after table() must not alter the open row. - handler.version = 53; - sender.uuidColumn("id", 1, 2).atNow(); - sender.flush(); - - new FrameReader(handler.awaitDataFrame()).uuidTable("events", 51, 52, 1, 2); - Assert.assertEquals("setters and completion must not discover schema", 1, - handler.describeRequests.get()); - } - }); - } - @Test public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { assertMemoryLeak(() -> { @@ -2074,7 +2016,6 @@ public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { } // The rejection neither looks up metadata nor splits the batch: the // next row joins the block pinned to version 52. - sender.table("events"); sender.uuidColumn("id", 3, 4).atNow(); long fsn = sender.flushAndGetSequence(); new FrameReader(handler.awaitDataFrame()).uuidTable("events", 51, 52, 1, 2, 3, 4); @@ -2204,86 +2145,6 @@ public void testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect() t }); } - @Test(timeout = 10_000) - public void testAsyncRowPreparedOfflineStaysLegacyWhenHandshakePrecedesFirstValue() throws Exception { - assertMemoryLeak(() -> { - int port = TestPorts.findUnusedPort(); - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port - + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" - + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" - + "auto_flush_bytes=0;auto_flush_interval=2147483646;" - + "close_flush_timeout_millis=0;"); - TestWebSocketServer server = null; - try { - Instant value = Instant.ofEpochSecond(3, 456_789_123); - sender.table("events"); - - SchemaHandler handler = new SchemaHandler( - QwpSchemaProtocol.RESULT_KNOWN, 73, 74, true, -1); - server = new TestWebSocketServer(handler, false, null, port); - server.setAdvertiseSchema(true); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); - - // The row contract was fixed by table(), before the connection existed. - sender.timestampColumn("ts", value).atNow(); - sender.flush(); - sender.table("events").timestampColumn("ts", value).atNow(); - sender.flush(); - - List frames = handler.awaitDataFrames(2); - new FrameReader(frames.get(0)).legacyTimestampTable("events", "ts", 3_456_789L); - new FrameReader(frames.get(1)).schemaTimestampTable( - "events", 73, 74, "ts", 3_456_789_123L); - Assert.assertEquals(1, handler.describeRequests.get()); - } finally { - sender.close(); - if (server != null) { - server.close(); - } - } - }); - } - - @Test(timeout = 10_000) - public void testCancellingAsyncRowPreparedOfflineDoesNotPinLegacyBatch() throws Exception { - assertMemoryLeak(() -> { - int port = TestPorts.findUnusedPort(); - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port - + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" - + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" - + "auto_flush_bytes=0;auto_flush_interval=2147483646;" - + "close_flush_timeout_millis=0;"); - TestWebSocketServer server = null; - try { - sender.table("events"); - - SchemaHandler handler = new SchemaHandler( - QwpSchemaProtocol.RESULT_KNOWN, 75, 76, true, -1); - server = new TestWebSocketServer(handler, false, null, port); - server.setAdvertiseSchema(true); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); - - sender.cancelRow(); - sender.table("events").timestampColumn( - "ts", Instant.ofEpochSecond(3, 456_789_123)).atNow(); - sender.flush(); - - new FrameReader(handler.awaitDataFrame()).schemaTimestampTable( - "events", 75, 76, "ts", 3_456_789_123L); - Assert.assertEquals(1, handler.describeRequests.get()); - } finally { - sender.close(); - if (server != null) { - server.close(); - } - } - }); - } - @Test public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exception { assertMemoryLeak(() -> { @@ -2328,29 +2189,30 @@ public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exceptio public void testSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); - handler.respondToDescribe = false; try (TestWebSocketServer server = schemaServer(handler); Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" - + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(1_000); + QwpTableBuffer b = ((QwpWebSocketSender) sender).getTableBuffer("b"); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + QwpTableBuffer events = ((QwpWebSocketSender) sender).getTableBuffer("events"); + handler.respondToDescribe = false; try { - sender.table("events"); + sender.table("b"); Assert.fail("expected schema negotiation deadline"); } catch (LineSenderSchemaException e) { Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); } - try { - sender.uuidColumn("id", 1, 2); - Assert.fail("failed table() must leave no open row"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("table() must be called")); - } - Assert.assertEquals("a setter after failed table() must not retry lookup", 1, - handler.describeRequests.get()); + Assert.assertEquals(1, events.getRowCount()); + Assert.assertEquals(0, b.getRowCount()); handler.respondToDescribe = true; - sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.uuidColumn("id", 3, 4).atNow(); + Assert.assertEquals(1, events.getRowCount()); + Assert.assertEquals(1, b.getRowCount()); sender.flush(); Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 9deef6d3d..cc35038ec 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -287,45 +287,23 @@ public void testCancelRowNoOpWithoutTable() throws Exception { } @Test - public void testEveryRowRequiresTableAndEmptyOpenRowRejectsLifecycleOperations() throws Exception { + public void testSelectedTableSupportsEmptyFlushAndImplicitRows() throws Exception { assertMemoryLeak(() -> { try (TestWebSocketServer server = startLegacyServer(); QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { - assertTableRequired(() -> sender.stringColumn("ignored", null)); - sender.table("t"); - try { - sender.table("t"); - Assert.fail("expected second table() to reject the open row"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("row is in progress")); - } - try { - sender.flush(); - Assert.fail("expected flush() to reject the empty open row"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("row is in progress")); - } - try { - sender.flushAndGetSequence(); - Assert.fail("expected flushAndGetSequence() to reject the empty open row"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("row is in progress")); - } - try { - sender.drain(1); - Assert.fail("expected drain() to reject the empty open row"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("row is in progress")); - } + sender.flush(); + sender.longColumn("v", 1).atNow(); + sender.longColumn("discarded", 2); sender.cancelRow(); - assertTableRequired(sender::atNow); - sender.table("t").atNow(); - assertTableRequired(sender::atNow); - sender.table("t").longColumn("v", 1).atNow(); - sender.cancelRow(); - assertTableRequired(() -> sender.longColumn("v", 2)); + sender.longColumn("v", 3).atNow(); + + sender.longColumn("v", 4); + sender.table("t"); // same-table selection does not disturb a partial row + sender.boolColumn("flag", true).atNow(); + + Assert.assertEquals(3, sender.getTableBuffer("t").getRowCount()); } }); } @@ -381,7 +359,7 @@ public void testDoubleColumnAfterCloseThrows() throws Exception { } @Test - public void testAutoFlushAppendFailureRetainsCommittedRowAndEndsOpenRow() throws Exception { + public void testFlushAppendFailureDoesNotLeaveMicrobatchBufferInUse() throws Exception { assertMemoryLeak(() -> { try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { })) { @@ -396,20 +374,17 @@ public void testAutoFlushAppendFailureRetainsCommittedRowAndEndsOpenRow() throws long minSegmentBytes = MmapSegment.HEADER_SIZE + MmapSegment.FRAME_HEADER_SIZE + 1; CursorSendEngine engine = new CursorSendEngine(null, minSegmentBytes, minSegmentBytes, 1L); try (QwpWebSocketSender sender = QwpWebSocketSender.connect( - "localhost", port, null, 1, 0, 0L, null, + "localhost", port, null, Integer.MAX_VALUE, 0, 0L, null, false, engine, 0L)) { + sender.table("t").longColumn("v", 1L).atNow(); + try { - sender.table("t").longColumn("v", 1L).atNow(); + sender.flushAndGetSequence(); Assert.fail("Expected LineSenderException"); } catch (LineSenderException e) { Assert.assertTrue(e.getMessage().contains("cursor SF append failed")); } - Assert.assertEquals("auto-flush failure must retain the committed row", 1, - sender.getTableBuffer("t").getRowCount()); - sender.table("t"); - sender.cancelRow(); - MicrobatchBuffer buffer0 = getMicrobatchBuffer(sender, "buffer0"); MicrobatchBuffer buffer1 = getMicrobatchBuffer(sender, "buffer1"); Assert.assertFalse( @@ -466,9 +441,7 @@ public void testIpv4ColumnStringNullReturnsThis() throws Exception { // way it round-trips as SQL NULL. The call must not throw and must // return the sender so chained builders keep working. assertMemoryLeak(() -> { - try (TestWebSocketServer server = startLegacyServer(); - QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { - sender.table("t"); + try (QwpWebSocketSender sender = createUnconnectedSender()) { Assert.assertSame(sender, sender.ipv4Column("addr", null)); } }); @@ -589,8 +562,9 @@ public void testNullArgsAfterCloseAllThrowSenderClosed() throws Exception { public void testNullArrayReturnsThis() throws Exception { assertMemoryLeak(() -> { try (QwpWebSocketSender sender = createUnconnectedSender()) { - assertTableRequired(() -> sender.doubleArray("x", (double[]) null)); - assertTableRequired(() -> sender.longArray("x", (long[]) null)); + // Null arrays should be no-ops and return sender + Assert.assertSame(sender, sender.doubleArray("x", (double[]) null)); + Assert.assertSame(sender, sender.longArray("x", (long[]) null)); } }); } @@ -896,15 +870,6 @@ private static MicrobatchBuffer getMicrobatchBuffer(QwpWebSocketSender sender, S return (MicrobatchBuffer) field.get(sender); } - private static void assertTableRequired(Runnable operation) { - try { - operation.run(); - Assert.fail("expected table() requirement"); - } catch (LineSenderException e) { - Assert.assertTrue(e.getMessage().contains("table() must be called")); - } - } - private static QwpWebSocketSender connectLegacySender( TestWebSocketServer server, int autoFlushRows, diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java index 18338e692..1aedb3ea1 100644 --- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolTest.java @@ -26,8 +26,6 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; -import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.impl.PooledSender; import io.questdb.client.impl.SenderPool; import io.questdb.client.test.tools.TestUtils; @@ -36,7 +34,6 @@ import java.lang.reflect.Field; import java.lang.reflect.Proxy; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; /** @@ -56,52 +53,6 @@ public class SenderPoolTest { private static final String DEAD_HTTP_CONFIG = "http::addr=127.0.0.1:1;protocol_version=2;auto_flush=off;"; - @Test - public void testQwpCloseCancelsEmptyAndPartialRowsBeforeReusingSlot() throws Exception { - TestUtils.assertMemoryLeak(() -> { - try (TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { - })) { - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - String config = "ws::addr=localhost:" + server.getPort() - + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" - + "close_flush_timeout_millis=0;"; - try (SenderPool pool = new SenderPool(config, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { - Sender first = pool.borrow(); - Object slot = slotOf(first); - Field delegateField = slot.getClass().getDeclaredField("delegate"); - delegateField.setAccessible(true); - QwpWebSocketSender delegate = (QwpWebSocketSender) delegateField.get(slot); - - first.table("events"); - first.close(); - Assert.assertEquals("table-only close must publish nothing", -1, - delegate.getCursorEngineForTesting().publishedFsn()); - - Sender second = pool.borrow(); - Assert.assertSame(slot, slotOf(second)); - Assert.assertThrows(LineSenderException.class, second::atNow); - second.table("events").longColumn("v", 1).atNow(); - second.table("events").longColumn("partial_only", 2); - second.close(); - Assert.assertEquals("close must publish the completed row", 0, - delegate.getCursorEngineForTesting().publishedFsn()); - Assert.assertEquals("the incomplete row must not retain its new column", 1, - delegate.getTableBuffer("events").getColumnCount()); - Assert.assertEquals(0, delegate.getTableBuffer("events").getRowCount()); - - try (Sender third = pool.borrow()) { - Assert.assertSame(slot, slotOf(third)); - Assert.assertThrows(LineSenderException.class, () -> third.longColumn("v", 3)); - third.table("events").longColumn("v", 3).atNow(); - } - Assert.assertEquals("the next borrower can publish its own row", 1, - delegate.getCursorEngineForTesting().publishedFsn()); - } - } - }); - } - @Test public void testBorrowReturnRecyclesSameDecorator() throws Exception { try (SenderPool pool = new SenderPool(DEAD_HTTP_CONFIG, 1, 1, 1_000, Long.MAX_VALUE, Long.MAX_VALUE)) { From 6ebb086ff8785e12499c92d549a4c27b2eda3d58 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 22 Sep 2026 10:51:45 +0200 Subject: [PATCH 32/51] Add schema_mode and schema_wait_millis knobs A schema-negotiated sender could not write to an unseen table during an outage: table() always ran a DESCRIBE and failed with SCHEMA_UNAVAILABLE after 30 s, breaking store-and-forward. The only workaround overloaded initial_connect_retry with the row wire contract. schema_mode=auto|strict|off (default auto) now owns that decision. auto uses the strict contract when a schema is pinned, cached, or a lookup answers while the wire is up, and otherwise writes the row legacy without waiting. strict waits out the budget and fails typed. off never looks schemas up. schema_wait_millis replaces the test-only constant. initial_connect_retry is startup-only again; the capability latch is unchanged. A pending schema batch is flushed before a legacy row starts a new one (one wire family per frame). The send loop exposes isWireUp() and peekSchema(), and abandons an in-flight lookup on disconnect under auto. --- README.md | 4 +- .../main/java/io/questdb/client/Sender.java | 152 +++++++- .../qwp/client/QwpWebSocketSender.java | 165 +++++++-- .../sf/cursor/CursorWebSocketSendLoop.java | 41 ++- .../sf/cursor/QwpSchemaCoordinator.java | 29 +- .../io/questdb/client/impl/ConfigSchema.java | 2 + .../LineSenderBuilderWebSocketTest.java | 49 +++ .../QwpSchemaSenderIntegrationTest.java | 348 ++++++++++++++++-- .../CursorWebSocketSchemaCoordinatorTest.java | 52 +++ .../test/impl/WsSenderConfigHonoredTest.java | 2 + 10 files changed, 770 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index e945c7770..c461a547a 100644 --- a/README.md +++ b/README.md @@ -555,7 +555,9 @@ Applied by the query pool to select and fail over between the nodes in the `addr | `client_id` | | Opaque client identifier surfaced server-side for observability | The ingest side also accepts store-and-forward and reconnection tuning keys (`auto_flush_*`, `initial_connect_retry`, -`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). +`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …) and +the schema-aware keys `schema_mode` (`auto`, `strict`, `off`) and `schema_wait_millis`; see +[docs/schema-aware-ingestion.md](docs/schema-aware-ingestion.md). `sf_durability=periodic` checkpoints mmap-published data in the background; `sf_sync_interval_millis` defaults to `5000` in that mode. The interval is a target cadence: JVM scheduling and storage-sync latency add to the actual loss window. Use `request_durable_ack=on` when end-to-end server durability is also required. See the diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 6810f23c1..2e3d9176a 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -765,8 +765,9 @@ default Sender shortColumn(CharSequence name, short value) { * to finalize the row. You can then start a new row by calling this method again. *
      * If you want to cancel the current row, you can call {@link #cancelRow()}. - * QWP WebSocket senders resolve the table's schema here when schema mode is active, so this call may wait for or - * fail schema discovery. Setters may resolve it again when an implicitly started row crosses a batch boundary. + * QWP WebSocket senders resolve the table's schema here when the server speaks schema-aware QWP, so this call may + * wait for or fail schema discovery; {@link SchemaMode} decides what happens when the schema is not obtainable. + * Setters may resolve it again when an implicitly started row crosses a batch boundary. * * @param table name of the table * @return this instance for method chaining @@ -821,17 +822,10 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Before schema support is confirmed, - * new rows use the legacy conversion contract; schema mismatches may - * therefore be reported later by the server, and values such as - * nanosecond timestamps use their legacy representation. If recovered - * data already requires schema support, writes instead wait for schema - * negotiation. A batch keeps the contract under which its first row - * was written: rows added to a pending legacy batch stay legacy even - * after support is confirmed. The first row after that batch flushes - * (explicitly or by auto-flush) and all later rows use strict schema - * lookup and validation, including across reconnects. Transport - * failures (unreachable or + * engine until the wire is up. {@link SchemaMode} decides whether a + * row written before the first upgrade waits for schema negotiation + * or uses the legacy conversion contract. Transport failures + * (unreachable or * dropped server) retry indefinitely and are never surfaced -- the * buffered rows are safe in SF and the server may still appear. A * terminal auth, upgrade or capability rejection on the initial @@ -856,6 +850,51 @@ enum InitialConnectMode { ASYNC } + /** + * Row-encoding contract of the WebSocket (QWP) sender when the server speaks + * schema-aware QWP. It decides what a row does when its table's schema is + * not obtainable at the moment the row starts: before the first upgrade + * completes under {@link InitialConnectMode#ASYNC}, while the wire is down + * during an outage, or while a schema lookup is still pending. It is + * independent of {@link InitialConnectMode}, which only governs how + * {@code fromConfig} behaves before the first successful upgrade. + *

        + *
      • {@link #AUTO} (default) — a row uses the strict schema contract + * whenever a schema is obtainable: from the batch's pinned snapshot, + * from the cache, or from a lookup that completes within + * {@code schema_wait_millis} while the wire is up. Otherwise the row + * uses the legacy conversion contract; schema unavailability never + * fails a row and the producer never waits on a wire that is down. + * Rows written that way keep store-and-forward's promise that an + * outage never blocks the producer, but the server, not the client, + * validates them, and values such as nanosecond timestamps use their + * legacy representation.
        • + *
      • {@link #STRICT} — once the server is known to speak schema-aware + * QWP, no row is ever encoded with the legacy contract. A row whose + * schema is not obtainable waits up to {@code schema_wait_millis}, + * across reconnects, and then fails with + * {@link LineSenderSchemaException} ({@code SCHEMA_UNAVAILABLE}). + * Under {@link InitialConnectMode#ASYNC} the first row also waits for + * the first upgrade, because the server's capability is unknown until + * then.
        • + *
      • {@link #OFF} — every row uses the legacy conversion contract and + * the sender never looks up schemas. The WebSocket upgrade is + * unchanged, so schema-encoded backlog recovered from a + * store-and-forward directory still replays.
        • + *
      + *

      + * A batch keeps the contract under which its first row was written: rows + * added to a pending legacy batch stay legacy even after a schema becomes + * obtainable, and a pending schema batch is published before a legacy row + * starts a new batch. A server that negotiates legacy QWP gets legacy rows + * in every mode. + */ + enum SchemaMode { + AUTO, + STRICT, + OFF + } + /** * Durability contract for the store-and-forward write path. Selects when * the SF segment file is fsynced; trades latency / throughput for @@ -1179,6 +1218,14 @@ public int getConnectTimeout() { private long reconnectMaxDurationMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean requestDurableAck; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; + // Row-encoding contract when a schema is not obtainable (WebSocket + // only). null means "not set explicitly", which build() resolves to + // AUTO. + private SchemaMode schemaMode = null; + // Upper bound for one schema wait: the initial handshake, if still + // pending, plus one DESCRIBE round trip. Reconnects consume the same + // budget. + private long schemaWaitMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean transactional; private String senderId = DEFAULT_SENDER_ID; // Per-append deadline for SF appendBlocking spin-then-throw. Used to @@ -1565,6 +1612,10 @@ public Sender build() { } else { actualInitialConnectMode = InitialConnectMode.OFF; } + SchemaMode actualSchemaMode = schemaMode == null ? SchemaMode.AUTO : schemaMode; + long actualSchemaWaitMillis = schemaWaitMillis == PARAMETER_NOT_SET_EXPLICITLY + ? QwpWebSocketSender.DEFAULT_SCHEMA_WAIT_MILLIS + : schemaWaitMillis; long actualDurableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis == DURABLE_ACK_KEEPALIVE_NOT_SET ? CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS @@ -1746,7 +1797,9 @@ public Sender build() { actualConnectionListenerInboxCapacity, actualMaxFrameRejections, actualPoisonMinEscalationWindowMillis, - actualCatchUpCapGapMinEscalationWindowMillis + actualCatchUpCapGapMinEscalationWindowMillis, + actualSchemaMode, + actualSchemaWaitMillis ); } catch (UnreplayableSlotException e) { // The one failure build() recovers from. The slot's frames reference ids @@ -2973,6 +3026,40 @@ public boolean isStoreAndForwardEnabled() { return sfDir != null; } + /** + * Row-encoding contract when a table's schema is not obtainable -- + * see {@link SchemaMode} for the value semantics. Default + * {@link SchemaMode#AUTO}. WebSocket transport only. + */ + public LineSenderBuilder schemaMode(SchemaMode mode) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_mode is only supported for WebSocket transport"); + } + if (mode == null) { + throw new LineSenderException("schema_mode cannot be null"); + } + this.schemaMode = mode; + return this; + } + + /** + * Upper bound for one schema wait: the first WebSocket upgrade, if it + * has not completed yet, plus one schema lookup round trip. Reconnects + * consume the same budget; they do not restart it. Default 30 s. At the + * bound, {@link SchemaMode#AUTO} writes the row with the legacy contract + * and {@link SchemaMode#STRICT} fails it. WebSocket transport only. + */ + public LineSenderBuilder schemaWaitMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_wait_millis is only supported for WebSocket transport"); + } + if (millis <= 0) { + throw new LineSenderException("schema_wait_millis must be > 0: ").put(millis); + } + this.schemaWaitMillis = millis; + return this; + } + /** * Per-call deadline for {@code Sender.flush()} spinning on a full * cursor segment ring waiting for ACKs to drain space. Default @@ -3913,6 +4000,18 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else { throw new LineSenderException("invalid initial_connect_retry [value=").put(sink).put(", allowed-values=[on, off, true, false, sync, async]]"); } + } else if (Chars.equals("schema_mode", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_mode is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "schema_mode"); + schemaMode(parseSchemaModeValue(sink)); + } else if (Chars.equals("schema_wait_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_wait_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "schema_wait_millis"); + schemaWaitMillis(parseLongValue(sink, "schema_wait_millis")); } else if (Chars.equals("sf_append_deadline_millis", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("sf_append_deadline_millis is only supported for WebSocket transport"); @@ -4176,6 +4275,15 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } + if (view.has("schema_wait_millis")) { + schemaWaitMillis(wsLong(view, v, "schema_wait_millis")); + } + s = view.getStr("schema_mode"); + if (s != null) { + v.clear(); + v.put(s); + schemaMode(parseSchemaModeValue(v)); + } if (view.has("sf_max_segment_bytes")) { storeAndForwardMaxSegmentBytes(wsSize(view, v, "sf_max_segment_bytes")); } @@ -4313,6 +4421,20 @@ private static int wsInt(ConfigView view, StringSink v, String key) { return parseIntValue(v, key); } + private static SchemaMode parseSchemaModeValue(CharSequence value) { + if (Chars.equalsIgnoreCase("auto", value)) { + return SchemaMode.AUTO; + } + if (Chars.equalsIgnoreCase("strict", value)) { + return SchemaMode.STRICT; + } + if (Chars.equalsIgnoreCase("off", value)) { + return SchemaMode.OFF; + } + throw new LineSenderException("invalid schema_mode [value=").put(value) + .put(", allowed-values=[auto, strict, off]]"); + } + private static long wsLong(ConfigView view, StringSink v, String key) { v.clear(); v.put(view.getStr(key)); @@ -4345,6 +4467,8 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("sf_max_total_bytes", sfMaxTotalBytes); m.put("sf_durability", sfDurability == null ? null : sfDurability.name()); m.put("sf_append_deadline_millis", sfAppendDeadlineMillis); + m.put("schema_mode", schemaMode == null ? null : schemaMode.name()); + m.put("schema_wait_millis", schemaWaitMillis); m.put("sf_sync_interval_millis", sfSyncIntervalMillis); m.put("close_flush_timeout_millis", closeFlushTimeoutMillis); m.put("durable_ack_keepalive_interval_millis", durableAckKeepaliveIntervalMillis); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1fd783dd8..a9f260800 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -151,7 +151,7 @@ public class QwpWebSocketSender implements Sender { // Upper bound for a row's first setter to wait on the initial handshake // and a schema DESCRIBE round trip. A healthy lookup takes milliseconds; // this only bounds a dead or still-connecting link. - private static final long DEFAULT_SCHEMA_WAIT_MILLIS = 30_000L; + public static final long DEFAULT_SCHEMA_WAIT_MILLIS = 30_000L; private static final String WRITE_PATH = "/write/v4"; // Yields the Authorization header value presented on each WebSocket upgrade. A constant for a // fixed token or Basic credential; for an httpTokenProvider it pulls a freshly refreshed token, @@ -398,6 +398,9 @@ public class QwpWebSocketSender implements Sender { // One-way owner capability: shared by foreground and every background // reconnect supplier created by this sender. private volatile boolean schemaRequired; + // Row-encoding contract when a schema is not obtainable; see + // resolveSchemaForCurrentTable(). + private Sender.SchemaMode schemaMode = Sender.SchemaMode.AUTO; private long schemaWaitMillis = DEFAULT_SCHEMA_WAIT_MILLIS; // Monotonic per-attempt counter snapshotted onto every connection event // fired from buildAndConnect. Counts every FOREGROUND endpoint try -- @@ -869,11 +872,51 @@ autoFlushIntervalNanos, fixedAuthHeader(authorizationHeader), catchUpCapGapMinEscalationWindowMillis); } + /** + * Overload with the default {@link Sender.SchemaMode#AUTO} schema contract + * and the default schema wait. + */ + public static QwpWebSocketSender connectWithCredentialSupplier( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + Supplier authorizationHeaderSupplier, + boolean requestDurableAck, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs, + int connectTimeoutMs, + SenderConnectionListener connectionListener, + int connectionListenerInboxCapacity, + int maxFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis + ) { + return connectWithCredentialSupplier(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, + autoFlushIntervalNanos, authorizationHeaderSupplier, requestDurableAck, cursorEngine, + closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, + connectionListenerInboxCapacity, maxFrameRejections, poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, Sender.SchemaMode.AUTO, DEFAULT_SCHEMA_WAIT_MILLIS); + } + /** * Master connect entry point — also accepts the poison-frame detector * threshold ({@code max_frame_rejections}): consecutive server-active * rejections of the same head-of-line frame, with no ack progress in - * between, before the loop escalates to a typed terminal. + * between, before the loop escalates to a typed terminal; and the schema + * contract ({@code schema_mode}) with its lookup budget + * ({@code schema_wait_millis}). *

      * Named apart from {@code connect} for the reason given on * {@link #connectWithCredentialSupplier(List, ClientTlsConfiguration, int, int, long, Supplier, @@ -903,7 +946,9 @@ public static QwpWebSocketSender connectWithCredentialSupplier( int connectionListenerInboxCapacity, int maxFrameRejections, long poisonMinEscalationWindowMillis, - long catchUpCapGapMinEscalationWindowMillis + long catchUpCapGapMinEscalationWindowMillis, + Sender.SchemaMode schemaMode, + long schemaWaitMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -925,6 +970,8 @@ public static QwpWebSocketSender connectWithCredentialSupplier( sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; + sender.schemaMode = schemaMode == null ? Sender.SchemaMode.AUTO : schemaMode; + sender.schemaWaitMillis = schemaWaitMillis; if (errorHandler != null) { sender.setErrorHandler(errorHandler); } @@ -2817,11 +2864,6 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } } - @TestOnly - public void setSchemaWaitMillisForTesting(long millis) { - schemaWaitMillis = millis; - } - @TestOnly public void setCursorSendLoopForTesting(CursorWebSocketSendLoop loop) { cursorSendLoop = loop; @@ -4301,6 +4343,10 @@ private void ensureConnected() { // the loop no longer fires a terminal budget-exhaustion event -- it // retries indefinitely.) cursorSendLoop.setConnectionDispatcher(connectionDispatcher); + // AUTO falls back to the legacy contract when the wire drops under a + // lookup, so the lookup must fail right away rather than ride the + // reconnect. STRICT keeps it pending for the replacement connection. + cursorSendLoop.setAbandonSchemaLookupOnDisconnect(schemaMode == Sender.SchemaMode.AUTO); cursorSendLoop.start(); } catch (Throwable t) { // start() (or dispatcher construction) failed after cursorSendLoop was @@ -4361,15 +4407,16 @@ private void ensureConnected() { /** * Selects the wire contract at an explicit table boundary or before the first - * effective value of an implicit row. In asynchronous initial-connect mode, - * fresh senders use the legacy contract until schema support is confirmed; - * recovered schema-framed data and senders that have already confirmed support - * retain the strict schema path. + * effective value of an implicit row. *

      * A table pins one schema snapshot per batch. Once the table holds pending rows, * later rows of the same batch validate against that snapshot without consulting * the cache; the first row after a flush or reset adopts whatever the cache holds * by then, which ACK schema feedback and reconnects keep current. + *

      + * When no snapshot is pinned, the configured {@link Sender.SchemaMode} decides + * between a schema lookup and the legacy contract; see + * {@link #resolveSchemaForCurrentTable()}. */ private QwpSchemaBinding bindingForEffectiveWrite() { QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); @@ -4382,20 +4429,15 @@ private QwpSchemaBinding bindingForEffectiveWrite() { } // A batch has one wire contract, because a QWP frame has one wire family. // While the pending batch holds legacy rows, later rows stay legacy even if - // a supporting handshake has landed since; the first row after the flush - // adopts schema mode. The upgrade is one-way, so a schema batch never - // reverts. - if ((pendingRowCount > 0 && isPendingBatchLegacy) - || (initialConnectMode == Sender.InitialConnectMode.ASYNC - && !cursorEngine.requiresSchema())) { - return null; + // a schema has become obtainable since; the first row after the flush + // consults the schema again. + if (pendingRowCount > 0 && isPendingBatchLegacy) { + return selectLegacyContract(); } - final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; - if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(deadlineNanos))) { - return null; + QwpSchemaResponse latest = resolveSchemaForCurrentTable(); + if (latest == null) { + return selectLegacyContract(); } - QwpSchemaResponse latest = cursorSendLoop.resolveSchema( - currentTableName, remainingSchemaMillis(deadlineNanos)); if (pinned != null && pinned.getTableId() == latest.getTableId() && pinned.getMetadataVersion() == latest.getMetadataVersion()) { @@ -4404,6 +4446,63 @@ private QwpSchemaBinding bindingForEffectiveWrite() { return installSchemaBinding(latest); } + /** + * Resolves the current table's schema under the configured + * {@link Sender.SchemaMode}, or returns {@code null} when the row must take the + * legacy contract: the mode is OFF, the server negotiated legacy QWP, or, in + * AUTO, the schema is not obtainable. + *

      + * STRICT waits for the first upgrade and then for the lookup within one + * {@code schemaWaitMillis} budget and lets a lookup failure propagate. AUTO + * never waits on a dead wire: before the first upgrade it takes the legacy + * contract outright, while the wire is down it consults only the cache, and + * a lookup that ends in {@code SCHEMA_UNAVAILABLE} (a drop under it, a timeout, + * a server-side unavailability) selects the legacy contract instead of failing + * the row. Access denial and unsupported responses still fail the row. + */ + private QwpSchemaResponse resolveSchemaForCurrentTable() { + if (schemaMode == Sender.SchemaMode.OFF) { + return null; + } + final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; + if (schemaMode == Sender.SchemaMode.STRICT) { + if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(deadlineNanos))) { + return null; + } + return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); + } + if (!cursorSendLoop.hasEverConnected() || !cursorEngine.requiresSchema()) { + return null; + } + if (!cursorSendLoop.isWireUp()) { + return cursorSendLoop.peekSchema(currentTableName); + } + try { + return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); + } catch (LineSenderSchemaException e) { + if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE) { + return null; + } + throw e; + } + } + + /** + * Commits the current table's next row to the legacy contract. A pending + * schema batch cannot take a legacy row, because a QWP frame has one wire + * family, so it is published first; a buffer still bound to an earlier batch's + * snapshot is unbound so the legacy setters lay out its columns afresh. + */ + private QwpSchemaBinding selectLegacyContract() { + if (pendingRowCount > 0 && !isPendingBatchLegacy) { + flushPendingRows(transactional); + } + if (currentTableBuffer.getSchemaBinding() != null) { + unbindCurrentTableBuffer(); + } + return null; + } + /** * Binds the row-less current table buffer to {@code latest}. A buffer that has * been bound before, or that a flushed legacy batch left with a column layout @@ -4411,16 +4510,24 @@ private QwpSchemaBinding bindingForEffectiveWrite() { * place; {@link QwpTableBuffer#clear()} drops both before rebinding. */ private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest) { - assert currentTableBuffer.getRowCount() == 0 : "schema binding replaced under pending rows"; if (currentTableBuffer.getSchemaBinding() != null || currentTableBuffer.getColumnCount() != 0) { - currentTableBuffer.clear(); - currentTableBufferSnapshotBytes = 0; - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; + unbindCurrentTableBuffer(); } return new QwpSchemaBinding(currentTableBuffer, latest); } + /** + * Drops the row-less current table buffer's binding and column layout so the + * next contract lays it out afresh. + */ + private void unbindCurrentTableBuffer() { + assert currentTableBuffer.getRowCount() == 0 : "table buffer unbound under pending rows"; + currentTableBuffer.clear(); + currentTableBufferSnapshotBytes = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + // Single home of the row-failure policy: roll the whole row back, then report // the failure as raised. A schema rejection is final for the batch's snapshot; // the sender never looks up fresh metadata from a failed setter. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index e440a6d8a..9057617b7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -436,6 +436,15 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // foreground sender's transient auth blip becomes a producer-fatal terminal -- the // exact failure this policy exists to prevent. private volatile boolean hasEverConnected; + // Producer-visible wire state. True from the moment swapClient installs a + // connection until the I/O thread enters the reconnect loop for it. The + // sender's schema contract selection reads it to decide between a lookup + // (wire up) and a cache-only probe (wire down). + private volatile boolean isWireUp; + // When true, connectLoop fails a pending schema lookup immediately instead of + // letting the replacement connection re-send it. Set by a sender whose schema + // mode can fall back to the legacy contract. + private volatile boolean shouldAbandonSchemaLookupOnDisconnect; // Cause of the outage the reconnect loop is currently riding out, or null once a // connect succeeds. Written by the I/O thread, read by the producer thread so a // backpressure or drain-timeout failure can NAME the reason the wire is not draining @@ -924,6 +933,7 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // hands null and lets the I/O thread connect — hasEverConnected // stays false until swapClient sees its first success. this.hasEverConnected = client != null; + this.isWireUp = client != null; if (engine.requiresSchema()) { if (reconnectFactory != null) { reconnectFactory.requireSchema(); @@ -1227,6 +1237,24 @@ public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMilli return schemaCoordinator.resolve(tableName, timeoutMillis, false, ioThread); } + /** + * Cache-only variant of {@link #resolveSchema}: the schema the coordinator + * already holds for the table, or {@code null}. Never sends a request and + * never waits, so a producer can call it while the wire is down. + */ + public QwpSchemaResponse peekSchema(CharSequence tableName) { + return schemaCoordinator.peek(tableName); + } + + /** + * Selects what happens to a schema lookup in flight when the connection + * drops: {@code true} fails it immediately, {@code false} (the default) keeps + * it pending with its original deadline for the replacement connection. + */ + public void setAbandonSchemaLookupOnDisconnect(boolean abandon) { + this.shouldAbandonSchemaLookupOnDisconnect = abandon; + } + /** * Evicts the named cache entry at admission and resolves it again through the I/O thread. * This producer-side API permits one outstanding lookup. The timeout covers the entire @@ -1572,6 +1600,15 @@ public boolean hasEverConnected() { return hasEverConnected; } + /** + * Whether a connection is currently installed. False before the first + * upgrade completes and from the moment the I/O thread enters the reconnect + * loop until {@code swapClient} installs the replacement. + */ + public boolean isWireUp() { + return isWireUp; + } + /** * Waits for the first completed WebSocket upgrade and returns whether that * handshake selected schema mode. A failed or still-pending handshake is @@ -1809,9 +1846,10 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM if (!running) { return; } + isWireUp = false; WebSocketClient failedClient = client; if (failedClient != null) { - schemaCoordinator.connectionLost(failedClient); + schemaCoordinator.connectionLost(failedClient, shouldAbandonSchemaLookupOnDisconnect); } if (reconnectFactory == null) { recordFatal(initial); @@ -2797,6 +2835,7 @@ private void swapClient(WebSocketClient newClient) { // dies inside the dictionary catch-up above was never fully // established, so the sender stays INITIALIZING and a subsequent // endpoint-policy rejection still latches the startup terminal. + this.isWireUp = true; this.hasEverConnected = true; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index b1281caa6..951bb95b8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -176,8 +176,22 @@ void applyFeedback(WebSocketResponse feedback) { } } - synchronized void connectionLost(WebSocketClient client) { - if (request != null && request.sentClient == client) { + /** + * Records that {@code client} died. With {@code abandon} false the producer's + * request keeps its deadline and the replacement connection re-sends it; with + * {@code abandon} true the request fails right away, so a producer that can + * fall back to the legacy contract does not sit out the rest of its budget on + * a wire that is known to be down. + */ + synchronized void connectionLost(WebSocketClient client, boolean abandon) { + if (request == null || request.done) { + return; + } + if (abandon) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "connection lost during schema lookup")); + return; + } + if (request.sentClient == client) { // Keep the producer's request and deadline; only its wire attempt ended. // A fresh ID also rejects old replies delivered on the replacement connection. request.sentClient = null; @@ -185,6 +199,17 @@ synchronized void connectionLost(WebSocketClient client) { } } + /** + * Cache-only lookup: the schema this coordinator already holds for the table, + * or {@code null}. Never sends a request and never waits. + */ + synchronized QwpSchemaResponse peek(CharSequence tableName) { + if (closed || cache == null) { + return null; + } + return cache.get(normalize(tableName)); + } + synchronized boolean invalidResponse(WebSocketClient client, long requestId) { // Zero means the malformed header could not identify a request. For an // identifiable reply, cancellation must share the liveness check's lock. diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index c9529a13e..613694d38 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -83,6 +83,8 @@ public final class ConfigSchema { str("reconnect_max_backoff_millis", Side.INGRESS); str("reconnect_max_duration_millis", Side.INGRESS); str("request_durable_ack", Side.INGRESS); + str("schema_mode", Side.INGRESS); + str("schema_wait_millis", Side.INGRESS); str("sender_id", Side.INGRESS); str("sf_append_deadline_millis", Side.INGRESS); str("sf_dir", Side.INGRESS); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 5d2b99763..9d4d3f137 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -252,6 +252,55 @@ public void testCatchUpCapGapMinEscalationWindowConfigStringNotSupportedForTcp() + "catch_up_cap_gap_min_escalation_window_millis=300000;")); } + @Test + public void testSchemaModeBuilderNotSupportedForTcp() { + assertThrows("schema_mode is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .schemaMode(Sender.SchemaMode.STRICT)); + } + + @Test + public void testSchemaModeConfigStringInvalidRejected() { + assertThrows("invalid schema_mode [value=maybe, allowed-values=[auto, strict, off]]", + () -> Sender.builder("ws::addr=localhost:9000;schema_mode=maybe;")); + } + + @Test + public void testSchemaModeConfigStringNotSupportedForTcp() { + assertThrows("schema_mode is only supported for WebSocket transport", + () -> Sender.builder("tcp::addr=localhost:9009;schema_mode=auto;")); + } + + @Test + public void testSchemaModeConfigStringValuesAreCaseInsensitive() { + Assert.assertEquals("AUTO", + Sender.builder("ws::addr=localhost:9000;schema_mode=Auto;").wsConfigSnapshotForTest().get("schema_mode")); + Assert.assertEquals("OFF", + Sender.builder("ws::addr=localhost:9000;schema_mode=OFF;").wsConfigSnapshotForTest().get("schema_mode")); + Assert.assertNull(Sender.builder("ws::addr=localhost:9000;").wsConfigSnapshotForTest().get("schema_mode")); + } + + @Test + public void testSchemaWaitMillisBuilderNotSupportedForTcp() { + assertThrows("schema_wait_millis is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .schemaWaitMillis(1000)); + } + + @Test + public void testSchemaWaitMillisConfigStringZeroRejected() { + assertThrows("schema_wait_millis must be > 0: 0", + () -> Sender.builder("ws::addr=localhost:9000;schema_wait_millis=0;")); + } + + @Test + public void testSchemaWaitMillisUnsetInSnapshot() { + Assert.assertEquals(-1L, + Sender.builder("ws::addr=localhost:9000;").wsConfigSnapshotForTest().get("schema_wait_millis")); + } + @Test public void testCatchUpCapGapMinEscalationWindowUnsetInSnapshot() { // wsConfigSnapshotForTest puts the raw field, not the resolved default (see diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index e9c6900cb..9723c0d70 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -1153,10 +1153,12 @@ public void testLongArrayRejectionKeepsPinnedSnapshotAfterSchemaChange() throws public void testLongArrayRejectionDoesNotDependOnDescribeAvailability() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); - try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_wait_millis=100;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { sender.table("events").uuidColumn("id", 1, 2).atNow(); handler.respondToDescribe = false; - ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(100); sender.table("events").stringColumn("partial", "discard"); try { sender.longArray("id", new long[]{99L}); @@ -1801,7 +1803,7 @@ public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throw } @Test - public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() throws Exception { + public void testStrictInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() throws Exception { assertMemoryLeak(() -> { int[] results = { QwpSchemaProtocol.RESULT_DENIED, @@ -1816,8 +1818,12 @@ public void testInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() thr for (int i = 0; i < results.length; i++) { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); handler.isSchemaFeedbackEnabled = true; + // STRICT surfaces every lookup failure; AUTO would write the + // RESULT_UNAVAILABLE case with the legacy contract instead. try (TestWebSocketServer server = schemaServer(handler); - Sender sender = sender(server)) { + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { sender.table("events").longColumn("id", 1).atNow(); handler.result = results[i]; // The pending batch never consults the server: the conflict is @@ -2186,17 +2192,17 @@ public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exceptio } @Test(timeout = 10_000) - public void testSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { + public void testStrictSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); try (TestWebSocketServer server = schemaServer(handler); Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() - + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + ";schema_mode=strict;schema_wait_millis=1000;" + + "initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + "close_flush_timeout_millis=0;")) { Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); - ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(1_000); QwpTableBuffer b = ((QwpWebSocketSender) sender).getTableBuffer("b"); sender.table("events").uuidColumn("id", 1, 2).atNow(); QwpTableBuffer events = ((QwpWebSocketSender) sender).getTableBuffer("events"); @@ -2221,13 +2227,13 @@ public void testSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() thr } @Test - public void testInterruptedSchemaLookupFailsTypedAndPreservesInterrupt() throws Exception { + public void testStrictInterruptedSchemaLookupFailsTypedAndPreservesInterrupt() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 93, 94); handler.respondToDescribe = false; try (TestWebSocketServer server = schemaServer(handler); Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() - + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + ";schema_mode=strict;initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -2261,31 +2267,20 @@ public void testInterruptedSchemaLookupFailsTypedAndPreservesInterrupt() throws } @Test(timeout = 10_000) - public void testAsyncRecoveredSchemaBacklogDoesNotFallBackToLegacy() throws Exception { + public void testStrictAsyncRecoveredSchemaBacklogWaitsForNegotiation() throws Exception { assertMemoryLeak(() -> { File sfRoot = temporaryFolder.newFolder("schema-required-recovery"); - SchemaHandler handler = new SchemaHandler( - QwpSchemaProtocol.RESULT_KNOWN, 95, 96, false); - int unavailablePort; - try (TestWebSocketServer server = schemaServer(handler)) { - String cfg = "ws::addr=localhost:" + server.getPort() - + ";sf_dir=" + sfRoot.getAbsolutePath() - + ";sender_id=recovery;close_flush_timeout_millis=0;"; - try (Sender sender = Sender.fromConfig(cfg)) { - sender.table("events").uuidColumn("id", 1, 2).atNow(); - sender.flush(); - Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] - & QwpConstants.FLAG_SCHEMA) != 0); - } - unavailablePort = TestPorts.findUnusedPort(); - } + int unavailablePort = publishUnackedSchemaBacklog(sfRoot, 95, 96); + // STRICT never takes the legacy contract once schema support is known, + // and the recovered backlog already requires it. With no server the + // wait runs out its configured budget and fails typed. try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + unavailablePort + ";sf_dir=" + sfRoot.getAbsolutePath() - + ";sender_id=recovery;initial_connect_retry=async;" + + ";sender_id=recovery;initial_connect_retry=async;schema_mode=strict;" + + "schema_wait_millis=250;" + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + "close_flush_timeout_millis=0;")) { - ((QwpWebSocketSender) sender).setSchemaWaitMillisForTesting(250); try { sender.table("events"); Assert.fail("recovered schema-required backlog fell back to legacy encoding"); @@ -2296,6 +2291,259 @@ public void testAsyncRecoveredSchemaBacklogDoesNotFallBackToLegacy() throws Exce }); } + @Test(timeout = 10_000) + public void testAutoAsyncRecoveredSchemaBacklogWritesLegacyOfflineAndReplaysBacklog() throws Exception { + assertMemoryLeak(() -> { + File sfRoot = temporaryFolder.newFolder("schema-required-recovery-auto"); + int port = publishUnackedSchemaBacklog(sfRoot, 97, 98); + + TestWebSocketServer server = null; + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;initial_connect_retry=async;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + // AUTO (the default) never waits on a wire that has not come up. The + // recovered backlog still pins the capability requirement for the + // reconnect, but this row takes the legacy contract right away. + sender.table("events").stringColumn("legacy_value", "A").atNow(); + + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 97, 98); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + sender.flush(); + + // The recovered schema frame replays first, then the legacy batch. + List frames = handler.awaitDataFrames(2); + FrameReader recovered = new FrameReader(frames.get(0)); + recovered.schemaTable("events", 97, 98, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, recovered.u8()); + Assert.assertEquals(1, recovered.i64()); + Assert.assertEquals(2, recovered.i64()); + recovered.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("events", "legacy_value", "A"); + Assert.assertEquals(0, handler.describeRequests.get()); + } finally { + if (server != null) { + server.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoOutageWritesUnseenTableLegacyAndNextBatchAdoptsSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 111, 112); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((first.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // The wire is down and "other" is not cached. AUTO must not wait out + // the 30 s default schema budget (the test timeout proves it); the row + // takes the legacy contract and queues in store-and-forward. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 113, 114); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + // The first row after the flush consults the schema again: "other" is + // looked up on the fresh connection and adopts the strict contract. + sender.table("other").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + + List frames = second.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyVarcharTable("other", "legacy_value", "A"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("other", 113, 114, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoOutagePublishesPendingSchemaBatchBeforeLegacyRow() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + // A schema row is pending and unflushed when the wire drops. + sender.table("events").uuidColumn("id", 1, 2).atNow(); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // A QWP frame has one wire family: the legacy row cannot join the + // pending schema batch, so that batch is published to store-and-forward + // first and the legacy row opens a new batch. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + Assert.assertEquals(0, ((QwpWebSocketSender) sender).getTableBuffer("events").getRowCount()); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + + List frames = second.awaitDataFrames(2); + FrameReader schema = new FrameReader(frames.get(0)); + schema.schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(1, schema.i64()); + Assert.assertEquals(2, schema.i64()); + schema.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("other", "legacy_value", "A"); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictOutageFailsUnseenTableWithinSchemaWaitAndRecovers() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 131, 132); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=strict;schema_wait_millis=250;", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((first.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // STRICT keeps waiting for the wire and fails typed at the budget. + try { + sender.table("other"); + Assert.fail("STRICT fell back to legacy encoding during an outage"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + // A cached table is still writable offline: "events" was pinned by the + // flushed batch's snapshot and the cache survives until reconnect. + sender.table("events").uuidColumn("id", 3, 4).atNow(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 131, 132); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.table("other").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + + List frames = second.awaitDataFrames(1); + Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(2, ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN) + .getShort(6) & 0xffff); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testSchemaModeOffNeverDescribesOnSchemaCapableServer() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test(timeout = 10_000) + public void testAutoAbandonsLookupWhenWireDropsUnderIt() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 151, 152); + first.respondToDescribe = false; + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + // A budget far beyond the test timeout: only the wire drop can end the wait. + Sender sender = outageSender(port, "schema_wait_millis=60000;", disconnected); + ExecutorService executor = Executors.newSingleThreadExecutor(); + TestWebSocketServer restarted = null; + try { + Future row = executor.submit(() -> { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + return null; + }); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (first.describeRequests.get() == 0) { + Assert.assertTrue("lookup was never sent", System.nanoTime() < deadline); + Thread.sleep(5); + } + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + // The connection loss fails the pending lookup and AUTO falls back to + // the legacy contract instead of sitting out the schema budget. + row.get(5, TimeUnit.SECONDS); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 151, 152); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + new FrameReader(second.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); + } finally { + executor.shutdownNow(); + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + @Test public void testResetDiscardsRowsAndKeepsCachedSnapshot() throws Exception { assertMemoryLeak(() -> { @@ -2410,12 +2658,58 @@ private static TestWebSocketServer legacyServer(SchemaHandler handler) throws Ex } } + /** + * A sender for the outage tests: no auto-flush, fast reconnect backoff, and a + * connection listener that trips {@code disconnected} when the wire drops. + * {@code extraConfig} is appended verbatim (e.g. {@code schema_mode=strict;}). + */ + private static Sender outageSender(int port, String extraConfig, CountDownLatch disconnected) { + return Sender.builder("ws::addr=localhost:" + port + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;" + extraConfig) + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build(); + } + + /** + * Writes one schema-encoded UUID row for "events" through a sender rooted at + * {@code sfRoot} against a server that never acknowledges data, so the frame + * stays in store-and-forward for a later sender to recover. Returns the port + * the server listened on, which is free again when this method returns. + */ + private static int publishUnackedSchemaBacklog(File sfRoot, int tableId, long version) throws Exception { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, tableId, version, false); + try (TestWebSocketServer server = schemaServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;close_flush_timeout_millis=0;"; + try (Sender sender = Sender.fromConfig(cfg)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + return server.getPort(); + } + } + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler) throws Exception { TestWebSocketServer server = new TestWebSocketServer(handler); server.setAdvertiseSchema(true); return start(server); } + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler, int port) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + return start(server); + } + private static TestWebSocketServer start(TestWebSocketServer server) throws Exception { try { server.start(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java index 0935943b6..e36bdf282 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java @@ -172,6 +172,58 @@ public void testTimeoutIgnoresLateReplyAndNextRequestSucceeds() throws Exception } } + @Test + public void testAbandonOnDisconnectFailsPendingLookupBeforeItsDeadline() throws Exception { + DescribeHandler handler = new DescribeHandler(false); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("abandon").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + loop.setAbandonSchemaLookupOnDisconnect(true); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertTrue(loop.isWireUp()); + // A budget far beyond the assertion window: only the drop can end it. + Future pending = executor.submit(() -> loop.resolveSchema("t", 60_000)); + Assert.assertTrue(handler.awaitFirstRequest()); + server.close(); + try { + pending.get(5, TimeUnit.SECONDS); + Assert.fail("lookup survived the connection loss"); + } catch (ExecutionException e) { + LineSenderSchemaException cause = (LineSenderSchemaException) e.getCause(); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, cause.getReason()); + Assert.assertTrue(cause.getMessage(), cause.getMessage().contains("connection lost")); + } + Assert.assertFalse(loop.isWireUp()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testPeekReturnsOnlyCachedSchema() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("peek").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertNull(loop.peekSchema("Trades")); + Assert.assertEquals(0, handler.requests.get()); + QwpSchemaResponse resolved = loop.resolveSchema("Trades", 5_000); + Assert.assertSame(resolved, loop.peekSchema("trades")); + Assert.assertEquals(1, handler.requests.get()); + } finally { + loop.close(); + } + } + } + @Test public void testUnnegotiatedConnectionIsUnsupported() throws Exception { DescribeHandler handler = new DescribeHandler(true); diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index 3c257827a..9483edaf3 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -69,6 +69,8 @@ public void testEveryIngressKeyIsHonored() { assertHonored("close_flush_timeout_millis=2500", "close_flush_timeout_millis", 2500L); assertHonored("durable_ack_keepalive_interval_millis=900", "durable_ack_keepalive_interval_millis", 900L); assertHonored("initial_connect_retry=async", "initial_connect_retry", "ASYNC"); + assertHonored("schema_mode=strict", "schema_mode", "STRICT"); + assertHonored("schema_wait_millis=1234", "schema_wait_millis", 1234L); assertHonored("reconnect_max_duration_millis=12345", "reconnect_max_duration_millis", 12345L); assertHonored("reconnect_initial_backoff_millis=111", "reconnect_initial_backoff_millis", 111L); assertHonored("reconnect_max_backoff_millis=2222", "reconnect_max_backoff_millis", 2222L); From 999b5355d6d73278263429a4028ec16a1e0e4682 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 22 Sep 2026 14:56:35 +0200 Subject: [PATCH 33/51] Fix legacy QWP failover with schema mode off Separate schema discovery from frame-driven replay requirements so a schema handshake does not prevent a legacy-only stream from failing over to an older server. Keep reconnect requirements local to each stream and negotiate schema support in OFF mode only for recovered schema backlog. Use current peer capability for AUTO lookups and preserve STRICT policy with independent negotiation history. Classify a lookup reaching a legacy replacement as unavailable so AUTO can use its existing fallback. Add failover, recovery, factory isolation and policy regression coverage. Four regressions fail before the fix and pass afterward. All 128 focused tests pass on Java 8 and Java 25. A broader run passes 332 of 333 tests; the remaining intermittent drainer failure also reproduces on the original code. Independent reviews pass. --- .../main/java/io/questdb/client/Sender.java | 11 +- .../qwp/client/QwpWebSocketSender.java | 22 +- .../client/sf/cursor/CursorSendEngine.java | 4 - .../sf/cursor/CursorWebSocketSendLoop.java | 36 +-- .../QwpSchemaSenderIntegrationTest.java | 244 +++++++++++++----- .../CursorWebSocketSchemaCoordinatorTest.java | 34 +-- .../CursorWebSocketSchemaLifecycleTest.java | 29 +++ .../sf/cursor/QwpSchemaReplayNetworkTest.java | 85 +++++- .../qwp/websocket/TestWebSocketServer.java | 7 + design/schema-aware-sender-journal.md | 16 ++ 10 files changed, 352 insertions(+), 136 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 2e3d9176a..b56551953 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -878,16 +878,17 @@ enum InitialConnectMode { * the first upgrade, because the server's capability is unknown until * then. *

    9. {@link #OFF} — every row uses the legacy conversion contract and - * the sender never looks up schemas. The WebSocket upgrade is - * unchanged, so schema-encoded backlog recovered from a - * store-and-forward directory still replays.
      10. + * the sender never looks up schemas. It requests schema support only + * when its stream must replay schema-encoded backlog recovered from + * a store-and-forward directory. Legacy-only streams can fail over + * to servers without schema support. * *

      * A batch keeps the contract under which its first row was written: rows * added to a pending legacy batch stay legacy even after a schema becomes * obtainable, and a pending schema batch is published before a legacy row - * starts a new batch. A server that negotiates legacy QWP gets legacy rows - * in every mode. + * starts a new batch. STRICT permits legacy rows only until a connection + * successfully negotiates schema support. */ enum SchemaMode { AUTO, diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index a9f260800..dc7c49bca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -395,9 +395,6 @@ public class QwpWebSocketSender implements Sender { private long reconnectMaxDurationMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS; private boolean requestDurableAck; - // One-way owner capability: shared by foreground and every background - // reconnect supplier created by this sender. - private volatile boolean schemaRequired; // Row-encoding contract when a schema is not obtainable; see // resolveSchemaForCurrentTable(). private Sender.SchemaMode schemaMode = Sender.SchemaMode.AUTO; @@ -3598,10 +3595,10 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); - // Every Sender connection probes the schema extension. Absence on the - // first successful handshake selects legacy mode; once confirmation is - // observed, the existing engine/factory latch makes it mandatory. - newClient.requestQwpSchema(); + // OFF still negotiates when this stream must replay schema frames. + if (schemaMode != Sender.SchemaMode.OFF || ctx.requiresSchema()) { + newClient.requestQwpSchema(); + } newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); if (cancellation != null) { // Publish the client we are about to block on so a @@ -4471,12 +4468,15 @@ private QwpSchemaResponse resolveSchemaForCurrentTable() { } return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); } - if (!cursorSendLoop.hasEverConnected() || !cursorEngine.requiresSchema()) { + if (!cursorSendLoop.hasEverConnected()) { return null; } if (!cursorSendLoop.isWireUp()) { return cursorSendLoop.peekSchema(currentTableName); } + if (!cursorSendLoop.isSchemaEnabled()) { + return null; + } try { return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); } catch (LineSenderSchemaException e) { @@ -5767,6 +5767,8 @@ private final class ReconnectSupplier implements CursorWebSocketSendLoop.Reconne */ private final java.util.function.BooleanSupplier abortCheck; private final String abortMessage; + // Each factory serves one engine; other streams can still use legacy peers. + private volatile boolean hasSchemaRequirement; private int previousIdx = -1; private ReconnectSupplier() { @@ -5804,12 +5806,12 @@ boolean isAborted() { } boolean requiresSchema() { - return schemaRequired; + return hasSchemaRequirement; } @Override public void requireSchema() { - schemaRequired = true; + hasSchemaRequirement = true; } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 21ac445b3..bb661529f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -993,10 +993,6 @@ public boolean requiresSchema() { return requiresSchema; } - public void requireSchema() { - requiresSchema = true; - } - private void latchSchemaRequirement(long payloadAddr, int payloadLen) { if (!requiresSchema && payloadAddr != 0 diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 9057617b7..0619c063e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -436,6 +436,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // foreground sender's transient auth blip becomes a producer-fatal terminal -- the // exact failure this policy exists to prevent. private volatile boolean hasEverConnected; + // STRICT row policy remembers successful schema negotiation independently of replay. + private volatile boolean hasNegotiatedSchema; // Producer-visible wire state. True from the moment swapClient installs a // connection until the I/O thread enters the reconnect loop for it. The // sender's schema contract selection reads it to decide between a lookup @@ -932,17 +934,13 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // already know we reached the server at least once. ASYNC startup // hands null and lets the I/O thread connect — hasEverConnected // stays false until swapClient sees its first success. + this.hasNegotiatedSchema = client != null && client.isQwpSchemaEnabled(); this.hasEverConnected = client != null; this.isWireUp = client != null; if (engine.requiresSchema()) { if (reconnectFactory != null) { reconnectFactory.requireSchema(); } - } else if (client != null && client.isQwpSchemaEnabled()) { - engine.requireSchema(); - if (reconnectFactory != null) { - reconnectFactory.requireSchema(); - } } // Adopt the engine's recovered orphaned-deferred-tail range (if any). // See the field docs on orphanSkipStartFsn/orphanSkipTipFsn; the @@ -1600,6 +1598,11 @@ public boolean hasEverConnected() { return hasEverConnected; } + public boolean isSchemaEnabled() { + WebSocketClient activeClient = client; + return activeClient != null && activeClient.isQwpSchemaEnabled(); + } + /** * Whether a connection is currently installed. False before the first * upgrade completes and from the moment the I/O thread enters the reconnect @@ -1610,9 +1613,9 @@ public boolean isWireUp() { } /** - * Waits for the first completed WebSocket upgrade and returns whether that - * handshake selected schema mode. A failed or still-pending handshake is - * never interpreted as a legacy peer. + * Waits for the first completed WebSocket upgrade and returns whether any + * installed connection has selected schema mode. A failed or still-pending + * handshake is never interpreted as a legacy peer. */ public boolean awaitInitialSchemaMode(long timeoutMillis) { long timeoutNanos = timeoutMillis > Long.MAX_VALUE / 1_000_000L @@ -1639,9 +1642,9 @@ public boolean awaitInitialSchemaMode(long timeoutMillis) { LockSupport.parkNanos(Math.min(50_000L, timeoutNanos - (System.nanoTime() - start))); } - // swapClient/constructor publishes the sticky engine requirement before - // hasEverConnected becomes visible. Read the authority, not client state. - return engine.requiresSchema(); + // Publish negotiation before hasEverConnected; recovered frames alone + // do not establish the first connection's row policy. + return hasNegotiatedSchema; } public boolean isRunning() { @@ -2786,12 +2789,6 @@ private void swapClient(WebSocketClient newClient) { } throw new QwpSchemaCapabilityMismatchException(); } - if (newClient.isQwpSchemaEnabled()) { - engine.requireSchema(); - if (reconnectFactory != null) { - reconnectFactory.requireSchema(); - } - } WebSocketClient old = this.client; this.client = newClient; schemaCoordinator.clearCache(); @@ -2835,6 +2832,9 @@ private void swapClient(WebSocketClient newClient) { // dies inside the dictionary catch-up above was never fully // established, so the sender stays INITIALIZING and a subsequent // endpoint-policy rejection still latches the startup terminal. + if (newClient.isQwpSchemaEnabled()) { + this.hasNegotiatedSchema = true; + } this.isWireUp = true; this.hasEverConnected = true; } @@ -3551,7 +3551,7 @@ private boolean trySendSchemaRequest() { return false; } if (!activeClient.isQwpSchemaEnabled()) { - schemaCoordinator.fail(request, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "server did not negotiate schema lookup support"); return true; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 9723c0d70..8df8c0e89 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -1917,16 +1917,19 @@ public void testUnsupportedSetterCannotBypassSchemaAndInvalidNameDoesNotRefresh( @Test public void testOldPeerUsesLegacyFrameAndNeverDescribes() throws Exception { assertMemoryLeak(() -> { - SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); - try (TestWebSocketServer server = legacyServer(handler); - Sender sender = sender(server)) { - sender.table("events").stringColumn("value", "legacy").atNow(); - sender.flush(); - byte[] frame = handler.awaitDataFrame(); - Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); - FrameReader reader = new FrameReader(frame); - reader.legacyVarcharTable("events", "value", "legacy"); - Assert.assertEquals(0, handler.describeRequests.get()); + for (boolean isStrict : new boolean[]{false, true}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + (isStrict ? "strict" : "auto") + ";close_flush_timeout_millis=0;")) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + FrameReader reader = new FrameReader(frame); + reader.legacyVarcharTable("events", "value", "legacy"); + Assert.assertEquals(0, handler.describeRequests.get()); + } } }); } @@ -2041,65 +2044,67 @@ public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { @Test public void testPendingLegacyBatchStaysLegacyAcrossUpgradeAndNextBatchAdoptsSchema() throws Exception { assertMemoryLeak(() -> { - int port = TestPorts.findUnusedPort(); - SchemaHandler legacyHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); - TestWebSocketServer legacy = new TestWebSocketServer(legacyHandler, false, null, port); - legacy.start(); - Assert.assertTrue(legacy.awaitStart(5, TimeUnit.SECONDS)); - CountDownLatch disconnected = new CountDownLatch(1); - Sender sender = Sender.builder("ws::addr=localhost:" + port - + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" - + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" - + "close_flush_timeout_millis=0;") - .connectionListener(event -> { - if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { - disconnected.countDown(); - } - }) - .build(); - TestWebSocketServer upgraded = null; - try { - sender.table("events").stringColumn("legacy_value", "A").atNow(); - legacy.close(); - Assert.assertTrue("sender did not observe the legacy connection closing", - disconnected.await(5, TimeUnit.SECONDS)); - sender.table("events").stringColumn("legacy_value", "B"); - SchemaHandler schemaHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 61, 62); - upgraded = new TestWebSocketServer(schemaHandler, false, null, port); - upgraded.setAdvertiseSchema(true); - upgraded.start(); - Assert.assertTrue(upgraded.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertTrue("schema-capable connection was not installed and servicing frames", - schemaHandler.awaitPong()); - sender.atNow(); - // A batch has one wire contract: the pending batch already holds - // legacy rows, so this row stays legacy although the supporting - // connection is up. The first row after the flush adopts schema mode. - sender.table("events").stringColumn("legacy_value", "C").atNow(); - sender.flush(); - sender.table("events").uuidColumn("id", 3, 4).atNow(); - sender.flush(); - List frames = schemaHandler.awaitDataFrames(2); - Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); - Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] - & QwpConstants.FLAG_DEFER_COMMIT); - Assert.assertTrue((frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] - & QwpConstants.FLAG_SCHEMA) != 0); - Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] - & QwpConstants.FLAG_DEFER_COMMIT); - new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B", "C"); - FrameReader schema = new FrameReader(frames.get(1)); - schema.schemaTable("events", 61, 62, 1, "id", QwpConstants.TYPE_UUID); - Assert.assertEquals(0, schema.u8()); - Assert.assertEquals(3, schema.i64()); - Assert.assertEquals(4, schema.i64()); - schema.eof(); - Assert.assertEquals(1, schemaHandler.describeRequests.get()); - } finally { - sender.close(); - legacy.close(); - if (upgraded != null) { - upgraded.close(); + for (boolean isStrict : new boolean[]{false, true}) { + int port = TestPorts.findUnusedPort(); + SchemaHandler legacyHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + TestWebSocketServer legacy = new TestWebSocketServer(legacyHandler, false, null, port); + legacy.start(); + Assert.assertTrue(legacy.awaitStart(5, TimeUnit.SECONDS)); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = Sender.builder("ws::addr=localhost:" + port + + ";schema_mode=" + (isStrict ? "strict" : "auto") + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;") + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build(); + TestWebSocketServer upgraded = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + legacy.close(); + Assert.assertTrue("sender did not observe the legacy connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + sender.table("events").stringColumn("legacy_value", "B"); + SchemaHandler schemaHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 61, 62); + upgraded = new TestWebSocketServer(schemaHandler, false, null, port); + upgraded.setAdvertiseSchema(true); + upgraded.start(); + Assert.assertTrue(upgraded.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed and servicing frames", + schemaHandler.awaitPong()); + sender.atNow(); + // A batch has one wire contract: the pending batch already holds + // legacy rows, so this row stays legacy although the supporting + // connection is up. The first row after the flush adopts schema mode. + sender.table("events").stringColumn("legacy_value", "C").atNow(); + sender.flush(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = schemaHandler.awaitDataFrames(2); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + Assert.assertTrue((frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B", "C"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("events", 61, 62, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, schemaHandler.describeRequests.get()); + } finally { + sender.close(); + legacy.close(); + if (upgraded != null) { + upgraded.close(); + } } } }); @@ -2483,6 +2488,106 @@ public void testStrictOutageFailsUnseenTableWithinSchemaWaitAndRecovers() throws }); } + @Test(timeout = 15_000) + public void testOffRetainedLegacyFramesFailOverFromCapableToLegacyPeer() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142, false); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=off;", disconnected); + TestWebSocketServer replacement = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + byte[] retained = first.awaitDataFrame(); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertArrayEquals(retained, second.awaitDataFrame()); + Assert.assertTrue(sender.drain(5_000)); + Assert.assertFalse(server.hasRequestedSchema()); + Assert.assertFalse(replacement.hasRequestedSchema()); + Assert.assertEquals(0, first.describeRequests.get()); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testStrictRemembersSchemaHandshakeWithoutPublishedFrames() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=strict;schema_wait_millis=100;", disconnected); + TestWebSocketServer replacement = null; + try { + Assert.assertTrue(first.awaitPong()); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + LineSenderSchemaException outage = Assert.assertThrows(LineSenderSchemaException.class, + () -> sender.table("events")); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, outage.getReason()); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertTrue("legacy replacement was not installed", second.awaitPong()); + LineSenderSchemaException downgrade = Assert.assertThrows(LineSenderSchemaException.class, + () -> sender.table("other")); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, downgrade.getReason()); + Assert.assertEquals(0, first.describeRequests.get()); + Assert.assertEquals(0, second.describeRequests.get()); + Assert.assertEquals(0, second.dataFrames.size()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testAutoDowngradesBeforePublishingSchemaFrames() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer replacement = null; + try { + // Lookup alone must not turn a schema-capable peer into a replay requirement. + sender.table("events"); + Assert.assertEquals(1, first.describeRequests.get()); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertTrue("legacy replacement was not installed", second.awaitPong()); + sender.table("other").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + new FrameReader(second.awaitDataFrame()).legacyVarcharTable("other", "legacy_value", "A"); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + @Test(timeout = 10_000) public void testSchemaModeOffNeverDescribesOnSchemaCapableServer() throws Exception { assertMemoryLeak(() -> { @@ -2495,6 +2600,7 @@ public void testSchemaModeOffNeverDescribesOnSchemaCapableServer() throws Except sender.flush(); new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); Assert.assertEquals(0, handler.describeRequests.get()); + Assert.assertFalse(server.hasRequestedSchema()); } }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java index e36bdf282..be54b0455 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java @@ -225,24 +225,26 @@ public void testPeekReturnsOnlyCachedSchema() throws Exception { } @Test - public void testUnnegotiatedConnectionIsUnsupported() throws Exception { - DescribeHandler handler = new DescribeHandler(true); - try (TestWebSocketServer server = new TestWebSocketServer(handler); - CursorSendEngine engine = new CursorSendEngine(temp.newFolder("legacy").getAbsolutePath(), 1 << 20)) { - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - try (WebSocketClient client = connect(server.getPort(), false)) { - CursorWebSocketSendLoop loop = loop(client, engine); - try { - loop.start(); - assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, - () -> loop.resolveSchema("t", 5_000)); - Assert.assertEquals(0, handler.requests.get()); - } finally { - loop.close(); + public void testUnnegotiatedConnectionIsUnavailable() throws Exception { + io.questdb.client.test.tools.TestUtils.assertMemoryLeak(() -> { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = new TestWebSocketServer(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("legacy").getAbsolutePath(), 1 << 20)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (WebSocketClient client = connect(server.getPort(), false)) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(0, handler.requests.get()); + } finally { + loop.close(); + } } } - } + }); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java index 7a99876c2..511fbbcf7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -181,6 +181,35 @@ public void testQueuedUnsentLookupSurvivesDisconnect() throws Exception { } } + @Test + public void testQueuedLookupBecomesUnavailableAtLegacyReplacement() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("queued-legacy-replacement"); + DisconnectingClient client = new DisconnectingClient()) { + server.setAdvertiseSchema(false); + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + reconnect.allow.countDown(); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Assert.assertTrue(client.receiving.await(5, TimeUnit.SECONDS)); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + client.disconnect.countDown(); + queued.join(); + assertUnavailable(queued.error.get()); + Assert.assertTrue(loop.isWireUp()); + Assert.assertFalse(loop.isSchemaEnabled()); + Assert.assertFalse(engine.requiresSchema()); + Assert.assertEquals(0, handler.requests); + } finally { + client.disconnect.countDown(); + } + } + } + } + @Test public void testReconnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { SchemaHandler handler = new SchemaHandler(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java index 5d4b1d4ab..4397e3b4b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -43,6 +43,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import io.questdb.client.test.tools.TestUtils; import java.io.File; import java.io.IOException; @@ -126,6 +127,54 @@ public void testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames() th assertNoQuarantine(root); } + @Test(timeout = 15_000) + public void testOffReplaysRecoveredSchemaFramesInForegroundAndOrphan() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (boolean isOrphan : new boolean[]{false, true}) { + File root = temp.newFolder(isOrphan ? "off-orphan" : "off-foreground"); + String slot = new File(root, isOrphan ? "ghost" : "default").getAbsolutePath(); + byte[][] expected = seedMixed(slot); + RecordingHandler handler = new RecordingHandler(true, "replay"); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + if (isOrphan) { + // The owner itself has no schema backlog; only this drainer opts in. + try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig( + "ws::addr=localhost:" + peer.getPort() + ";schema_mode=off;")) { + Assert.assertFalse(peer.hasRequestedSchema()); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, SEGMENT_SIZE, 4L * SEGMENT_SIZE, + sender.newBackgroundReconnectFactory(() -> false), + 5_000, 1, 4, false, 0); + Thread thread = new Thread(drainer, "off-schema-orphan"); + thread.start(); + try { + thread.join(5_000); + Assert.assertFalse("drainer did not finish", thread.isAlive()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + } finally { + drainer.requestStop(); + thread.join(5_000); + } + } + } else { + try (Sender sender = Sender.fromConfig(config(peer.getPort(), root, false) + + "schema_mode=off;")) { + Assert.assertTrue(sender.drain(5_000)); + } + } + Assert.assertTrue(peer.hasRequestedSchema()); + } + Assert.assertEquals(2, handler.frames.size()); + Assert.assertArrayEquals(expected[0], handler.frames.get(0)); + Assert.assertArrayEquals(expected[1], handler.frames.get(1)); + assertNoQuarantine(root); + } + }); + } + @Test public void testPublicSenderSchemaFrameSurvivesRestartByteForByte() throws Exception { File root = temp.newFolder("public-sender-restart"); @@ -288,15 +337,17 @@ public void testInvalidSchemaColumnsDoNotAdvanceRecoveredBacklog() throws Except } @Test - public void testSchemaRequirementIsSharedAcrossForegroundAndBackgroundFactories() throws Exception { - RecordingHandler oldHandler = new RecordingHandler(false, "unused"); - try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { - oldPeer.start(); - Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); - assertSchemaRequirementShared(oldPeer.getPort(), true); - assertSchemaRequirementShared(oldPeer.getPort(), false); - Assert.assertEquals(0, oldHandler.frames.size()); - } + public void testSchemaRequirementIsIsolatedAcrossForegroundAndBackgroundFactories() throws Exception { + TestUtils.assertMemoryLeak(() -> { + RecordingHandler oldHandler = new RecordingHandler(false, "unused"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + assertSchemaRequirementIsolated(oldPeer.getPort(), true); + assertSchemaRequirementIsolated(oldPeer.getPort(), false); + Assert.assertEquals(0, oldHandler.frames.size()); + } + }); } private static void assertNoQuarantine(File root) { @@ -336,7 +387,7 @@ private static void assertRetained(String slot, long publishedFsn) { } } - private static void assertSchemaRequirementShared(int port, boolean foregroundFirst) throws Exception { + private static void assertSchemaRequirementIsolated(int port, boolean foregroundFirst) throws Exception { try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig( "ws::addr=localhost:" + port + ';')) { CursorWebSocketSendLoop.ReconnectFactory foreground; @@ -345,12 +396,18 @@ private static void assertSchemaRequirementShared(int port, boolean foregroundFi foreground = sender.newReconnectFactory(); background = sender.newBackgroundReconnectFactory(() -> false); foreground.requireSchema(); - assertSchemaMismatch(background); + assertSchemaMismatch(foreground); + try (WebSocketClient client = background.reconnect()) { + Assert.assertFalse(client.isQwpSchemaEnabled()); + } } else { background = sender.newBackgroundReconnectFactory(() -> false); foreground = sender.newReconnectFactory(); background.requireSchema(); - assertSchemaMismatch(foreground); + assertSchemaMismatch(background); + try (WebSocketClient client = foreground.reconnect()) { + Assert.assertFalse(client.isQwpSchemaEnabled()); + } } } } @@ -359,9 +416,9 @@ private static void assertSchemaMismatch(CursorWebSocketSendLoop.ReconnectFactor WebSocketClient client = null; try { client = factory.reconnect(); - Assert.fail("an old peer must be rejected after either factory requires schema"); + Assert.fail("an old peer must be rejected by the schema-required factory"); } catch (QwpSchemaCapabilityMismatchException expected) { - // Expected: the requirement belongs to the sender owner, not one factory instance. + // Expected: the requirement belongs to this factory's stream. } finally { if (client != null) { client.close(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 788e772b2..8f0cf9120 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -126,6 +126,7 @@ public class TestWebSocketServer implements Closeable { // several frames. Live-updatable via setAdvertisedMaxBatchSize(). private volatile int advertisedMaxBatchSize; private volatile boolean advertiseSchema; + private volatile boolean hasRequestedSchema; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -231,6 +232,10 @@ public int getPort() { return port; } + public boolean hasRequestedSchema() { + return hasRequestedSchema; + } + /** * Number of handshakes the server has completed over its lifetime * (monotonic; never decreases when clients disconnect). @@ -622,6 +627,8 @@ private boolean performHandshake() throws IOException { return false; } capturedAuthHeaders.add(authorization); + hasRequestedSchema |= request.toString().toLowerCase(java.util.Locale.ROOT) + .contains("x-qwp-request-schema: true\r\n"); // Read-path reject: drop the egress upgrade before the 101 so the // query pool's connect fails fast, while ingest write-path upgrades diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md index 322acecdd..3f3421b29 100644 --- a/design/schema-aware-sender-journal.md +++ b/design/schema-aware-sender-journal.md @@ -8057,3 +8057,19 @@ The poll bit is already on `master`, while `FLAG_SCHEMA` is unreleased, so `QwpConstants`. This supersedes the `FLAG_SCHEMA=0x02` value recorded in design revision 13. The trade-off: store-and-forward frames persisted by an earlier build of this branch carry `0x02` and no longer replay as schema frames. + +## OFF failover and per-stream replay requirements + +Schema negotiation no longer marks an engine as requiring schema-framed replay. +The engine latches that requirement from live or recovered frame content, before +publication, and retains it for its lifetime even after those frames are acknowledged. +Each reconnect factory carries only its own engine's requirement. OFF requests +schema support only to replay such a backlog; new rows remain legacy. + +AUTO chooses lookup eligibility from the installed connection and uses the cache +while disconnected. A pending lookup reaching a legacy replacement fails with +SCHEMA_UNAVAILABLE, allowing AUTO's existing fallback. STRICT separately remembers +a successfully installed schema connection, so a handshake alone still prevents +new legacy rows during an outage or downgrade. Neither policy memory nor another +stream's backlog rejects a legacy endpoint. Recovered schema backlog still waits +for a supporting connection before replay. From f9ab44dd33e44d746b4f9a0ee1b621d7d4a7fe06 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 22 Sep 2026 15:25:16 +0200 Subject: [PATCH 34/51] Allow AUTO fallback for oversized QWP schemas Use the existing legacy fallback when AUTO schema lookup reports an unsupported or oversized response. This preserves partial-column writes to tables wider than the schema discovery limit. Keep STRICT lookup failures and local conversion errors unchanged. Add regressions for legacy fallback and publishing a pending schema batch before starting a legacy row. Validation: 108 focused tests passed on Java 8 and 86 schema sender integration tests passed on Java 25. --- .../qwp/client/QwpWebSocketSender.java | 8 ++-- .../QwpSchemaSenderIntegrationTest.java | 44 ++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index dc7c49bca..62a34a9c7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4454,8 +4454,9 @@ private QwpSchemaBinding bindingForEffectiveWrite() { * never waits on a dead wire: before the first upgrade it takes the legacy * contract outright, while the wire is down it consults only the cache, and * a lookup that ends in {@code SCHEMA_UNAVAILABLE} (a drop under it, a timeout, - * a server-side unavailability) selects the legacy contract instead of failing - * the row. Access denial and unsupported responses still fail the row. + * a server-side unavailability) or {@code UNSUPPORTED_FEATURE} (an oversized + * or unsupported schema response) selects the legacy contract instead of failing + * the row. Access denial still fails the row. */ private QwpSchemaResponse resolveSchemaForCurrentTable() { if (schemaMode == Sender.SchemaMode.OFF) { @@ -4480,7 +4481,8 @@ private QwpSchemaResponse resolveSchemaForCurrentTable() { try { return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); } catch (LineSenderSchemaException e) { - if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE) { + if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE + || e.getReason() == LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) { return null; } throw e; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 8df8c0e89..8fb7ca4c6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -1802,6 +1802,48 @@ public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throw }); } + @Test + public void testAutoUnavailableSchemaUsesLegacyFrame() throws Exception { + assertMemoryLeak(() -> { + for (int result : new int[]{QwpSchemaProtocol.RESULT_UNAVAILABLE, QwpSchemaProtocol.RESULT_TOO_LARGE}) { + SchemaHandler handler = new SchemaHandler(result, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "value", "legacy"); + } + } + }); + } + + @Test + public void testAutoTooLargeSchemaPublishesPendingSchemaBatchBeforeLegacyRow() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.result = QwpSchemaProtocol.RESULT_TOO_LARGE; + + // The new table cannot join the pending schema batch: publish it + // before the legacy row starts a new frame. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(2); + FrameReader schema = new FrameReader(frames.get(0)); + schema.schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(1, schema.i64()); + Assert.assertEquals(2, schema.i64()); + schema.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("other", "legacy_value", "A"); + } + }); + } + @Test public void testStrictInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() throws Exception { assertMemoryLeak(() -> { @@ -1819,7 +1861,7 @@ public void testStrictInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); handler.isSchemaFeedbackEnabled = true; // STRICT surfaces every lookup failure; AUTO would write the - // RESULT_UNAVAILABLE case with the legacy contract instead. + // RESULT_UNAVAILABLE and RESULT_TOO_LARGE cases with the legacy contract instead. try (TestWebSocketServer server = schemaServer(handler); Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" From a12ae239bd058b04470fcf965d1080a452225188 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Tue, 22 Sep 2026 16:08:51 +0200 Subject: [PATCH 35/51] Exclude arrays from GEOHASH setter dispatch Guard the string and GEOHASH setters against array type metadata sharing the GEOHASH flag. Unsupported scalar writes now produce the existing schema error instead of an NPE or an unintended NULL. Preserve explicit string null writes to array columns. Add regression coverage across all 32 array ranks. Validate 142 schema-binding tests on Java 8 and 61 surrounding tests on Java 25. --- .../qwp/protocol/QwpSchemaBinding.java | 4 +- .../QwpSchemaBindingDoubleArrayTest.java | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index dedf26a6f..f1bff987b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -396,7 +396,7 @@ public QwpSchemaBinding geoHashColumn(CharSequence name, long bits, int precisio return this; } long value = bits & ((1L << precisionBits) - 1L); - if (ColumnType.isGeoHash(targetType)) { + if (!ColumnType.isArray(targetType) && ColumnType.isGeoHash(targetType)) { int targetBits = ColumnType.getGeoHashBits(targetType); if (targetBits != precisionBits) { throw unsupported(name, "GEOHASH", targetType, @@ -601,7 +601,7 @@ public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { if (column == null) { return this; } - if (ColumnType.isGeoHash(targetType)) { + if (!ColumnType.isArray(targetType) && ColumnType.isGeoHash(targetType)) { int precision = ColumnType.getGeoHashBits(targetType); column.initGeoHashPrecision(precision); if (value == null || value.length() == 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java index 50278f69f..cc69a925c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java @@ -193,6 +193,60 @@ public void testRejectsWrongElementTypeWeakRankParametersAndDesignatedColumn() { } } + @Test + public void testRejectsStringValuesForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("a", "zz")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("a", "")); + } + } + }); + } + + @Test + public void testRejectsGeoHashValuesForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.geoHashColumn("a", 1023L, 10)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.geoHashColumn("a", "zz")); + } + } + }); + } + + @Test + public void testStringNullUsesArrayNullWireForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + binding.stringColumn("a", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + columnDef(reader, "a"); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + } + }); + } + private static void array(QwpTestWireReader reader, int[] shape, double... values) { Assert.assertEquals(0, reader.u8()); rawArray(reader, shape.length, shape, values); From 358db5f7e0aafdc8dfba691250c368c0fdfd322c Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 12:24:41 +0200 Subject: [PATCH 36/51] Release schema requirement once frames are acked CursorSendEngine latched requiresSchema on the first FLAG_SCHEMA frame and never cleared it, and ReconnectSupplier kept a second one-way copy. After one schema batch was acked, a rollback or failover to an older server left the sender retrying forever while its SF backlog filled, until a JVM restart recomputed the flag from unacked frames. The engine now tracks the FSN of the newest schema-framed frame and requiresSchema() compares it with ackedFsn, so the requirement lasts exactly while such a frame is unacked. The producer records the FSN before publishing the frame, which keeps the post-publication check in trySendOne effective. Recovery seeds the same FSN. Reconnect factories take setSchemaRequired(boolean) and every caller refreshes it from the engine before each attempt. Orphan-tail retirement no longer requires a schema-capable connection. Retirement transmits nothing, and retiring schema-framed tail frames now clears the requirement they imposed, so the background drainer reclaims such a slot before opening a socket. Tests cover the requirement following acks, a live downgrade from a schema peer to a legacy peer, local retirement of a schema orphan tail, and a drainer that retires it without connecting. All fail with the old behaviour restored; 2284 QWP client tests pass. --- .../qwp/client/QwpWebSocketSender.java | 12 +- .../client/sf/cursor/BackgroundDrainer.java | 6 +- .../client/sf/cursor/CursorSendEngine.java | 44 ++++--- .../sf/cursor/CursorWebSocketSendLoop.java | 20 ++-- .../sf/cursor/RecoveredFrameAnalysis.java | 19 +-- .../sf/cursor/CursorSendEngineTest.java | 40 +++++++ .../CursorWebSocketSchemaLifecycleTest.java | 2 +- .../sf/cursor/QwpSchemaReplayNetworkTest.java | 110 ++++++++++++++++-- .../sf/cursor/RecoveredFrameAnalysisTest.java | 8 +- 9 files changed, 200 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 62a34a9c7..b4faf3b43 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4261,9 +4261,7 @@ private void ensureConnected() { connectionListener, connectionListenerInboxCapacity); } CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory(); - if (cursorEngine.requiresSchema()) { - reconnectFactory.requireSchema(); - } + reconnectFactory.setSchemaRequired(cursorEngine.requiresSchema()); switch (initialConnectMode) { case SYNC: client = CursorWebSocketSendLoop.connectWithRetry( @@ -5770,7 +5768,7 @@ private final class ReconnectSupplier implements CursorWebSocketSendLoop.Reconne private final java.util.function.BooleanSupplier abortCheck; private final String abortMessage; // Each factory serves one engine; other streams can still use legacy peers. - private volatile boolean hasSchemaRequirement; + private volatile boolean isSchemaRequired; private int previousIdx = -1; private ReconnectSupplier() { @@ -5808,12 +5806,12 @@ boolean isAborted() { } boolean requiresSchema() { - return hasSchemaRequirement; + return isSchemaRequired; } @Override - public void requireSchema() { - hasSchemaRequirement = true; + public void setSchemaRequired(boolean isRequired) { + isSchemaRequired = isRequired; } @Override diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index b6f2a17e8..5833ffbd2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -482,6 +482,9 @@ private WebSocketClient connectWithDurableAckRetry(CursorSendEngine schemaEngine // to the deadline (which would otherwise busy-loop once past it). boolean boundedByBudget = false; try { + if (schemaEngine != null) { + clientFactory.setSchemaRequired(schemaEngine.requiresSchema()); + } WebSocketClient connected = clientFactory.reconnect(); if (connected != null && schemaEngine != null && schemaEngine.requiresSchema() && !connected.isQwpSchemaEnabled()) { @@ -1072,9 +1075,6 @@ public void run() { return; } engineForTesting = engine; - if (engine.requiresSchema()) { - clientFactory.requireSchema(); - } if (logicalSlotLock != null) { logicalSlotLock.close(); logicalSlotLock = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index bb661529f..1215446d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -175,7 +175,12 @@ public final class CursorSendEngine implements QuietCloseable { // because both producer seeding and every recycled send loop need the same // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. private final RecoveredFrameAnalysis recoveredFrameAnalysis; - private volatile boolean requiresSchema; + // FSN of the newest FLAG_SCHEMA frame in the ring; -1 when none. A schema + // requirement exists only while this frame is unacked, so a peer without + // schema support becomes usable again once the server acks it. Written by + // the single producer thread (and by the constructor from recovery), read + // by the I/O thread. + private volatile long maxSchemaFsn = -1L; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -822,8 +827,9 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.watermark = watermarkInProgress; this.persistedSymbolDict = persistedDictInProgress; this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; - this.requiresSchema = recoveredFrameAnalysisInProgress != null - && recoveredFrameAnalysisInProgress.requiresSchema(); + if (recoveredFrameAnalysisInProgress != null) { + this.maxSchemaFsn = recoveredFrameAnalysisInProgress.maxSchemaFsn(); + } } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -920,7 +926,7 @@ MmapSegment pinActiveSegment() { * is wedged" failures (server down, slow disk, etc.) from the user. */ public long appendBlocking(long payloadAddr, int payloadLen) { - latchSchemaRequirement(payloadAddr, payloadLen); + trackSchemaFrame(payloadAddr, payloadLen); long fsn = ring.appendOrFsn(payloadAddr, payloadLen); if (fsn >= 0) return fsn; if (fsn == SegmentRing.PAYLOAD_TOO_LARGE) { @@ -964,7 +970,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) { * {@code SegmentRing.BACKPRESSURE_*} / {@code PAYLOAD_*} sentinels. */ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos) { - latchSchemaRequirement(payloadAddr, payloadLen); + trackSchemaFrame(payloadAddr, payloadLen); long fsn = ring.appendOrFsn(payloadAddr, payloadLen); if (fsn >= 0) { return fsn; @@ -989,20 +995,25 @@ public void checkDurability() { ring.checkDurability(); } + /** + * True while an unacked frame in the ring carries {@code FLAG_SCHEMA} and + * therefore must only reach a peer that negotiated schema framing. + */ public boolean requiresSchema() { - return requiresSchema; + return maxSchemaFsn > ring.ackedFsn(); } - private void latchSchemaRequirement(long payloadAddr, int payloadLen) { - if (!requiresSchema - && payloadAddr != 0 + private void trackSchemaFrame(long payloadAddr, int payloadLen) { + if (payloadAddr != 0 && payloadLen >= QwpConstants.HEADER_SIZE && io.questdb.client.std.Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE && (io.questdb.client.std.Unsafe.getUnsafe().getByte( payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA) != 0) { // Publish the capability requirement before ring.append publishes - // the frame to the I/O consumer. - requiresSchema = true; + // the frame to the I/O consumer. The single producer makes the + // next FSN exact; if the append fails, the frame that eventually + // takes this FSN keeps the requirement until it is acked. + maxSchemaFsn = ring.publishedFsn() + 1L; } } @@ -1899,23 +1910,18 @@ public long recoveredOrphanTipFsn() { /** * Retires a recovered deferred tail once every frame below it is ACKed. * The operation is local and idempotent: no wire sequence ever referred - * to these aborted-transaction frames. + * to these aborted-transaction frames, so it needs no peer capability. + * Retiring schema-framed tail frames also drops the schema requirement + * they imposed. * * @return true if no orphan tail remains, false if lower frames still need * server ACKs */ public boolean retireRecoveredOrphanTailIfReady() { - return retireRecoveredOrphanTailIfReady(false); - } - - public boolean retireRecoveredOrphanTailIfReady(boolean schemaConfirmed) { long orphanTip = recoveredOrphanTipFsn; if (orphanTip < 0L) { return true; } - if (requiresSchema && !schemaConfirmed) { - return false; - } long orphanStart = recoveredCommitBoundaryFsn + 1L; if (ackedFsn() < orphanStart - 1L) { return false; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 0619c063e..5db8cf52d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -937,11 +937,6 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, this.hasNegotiatedSchema = client != null && client.isQwpSchemaEnabled(); this.hasEverConnected = client != null; this.isWireUp = client != null; - if (engine.requiresSchema()) { - if (reconnectFactory != null) { - reconnectFactory.requireSchema(); - } - } // Adopt the engine's recovered orphaned-deferred-tail range (if any). // See the field docs on orphanSkipStartFsn/orphanSkipTipFsn; the // BackgroundDrainer path builds its loop through this same @@ -1910,9 +1905,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { - if (engine.requiresSchema()) { - reconnectFactory.requireSchema(); - } + reconnectFactory.setSchemaRequired(engine.requiresSchema()); WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { @@ -3789,8 +3782,7 @@ private boolean tryRetireOrphanTail() { if (orphanSkipTipFsn < 0) { return true; } - if (!engine.retireRecoveredOrphanTailIfReady( - client != null && client.isQwpSchemaEnabled())) { + if (!engine.retireRecoveredOrphanTailIfReady()) { return false; } orphanSkipStartFsn = -1L; @@ -3829,7 +3821,13 @@ public enum ReconnectPolicy { public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; - default void requireSchema() { + /** + * Tells the factory whether the engine currently holds unacked + * schema-framed frames, so it requests schema framing and rejects peers + * without it. Callers refresh this before each attempt; the send loop + * re-checks the engine when it installs the client. + */ + default void setSchemaRequired(boolean isRequired) { } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index c4fe33d8b..9fdd0e2ba 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -65,7 +65,8 @@ final class RecoveredFrameAnalysis implements QuietCloseable { private int runningRawCount; private int runningRawLen; private long runningCoverage; - private boolean requiresSchema; + // FSN of the newest FLAG_SCHEMA frame visited; -1 when none. + private long maxSchemaFsn = -1L; RecoveredFrameAnalysis(int baseline, long ackedFsn) { this.baseline = baseline; @@ -82,10 +83,10 @@ void accept(long fsn, long payload, int payloadLen) { byte flags = isQwp ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS) : 0; - if (fsn > ackedFsn && isQwp && (flags & QwpConstants.FLAG_SCHEMA) != 0) { - // Conservative by design: retained deferred tails may still be - // replay-owned until the engine's orphan-tail retirement runs. - requiresSchema = true; + if (isQwp && (flags & QwpConstants.FLAG_SCHEMA) != 0) { + // Includes a retained deferred tail: it may still be replay-owned + // until the engine's orphan-tail retirement acknowledges it. + maxSchemaFsn = fsn; } if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) { foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen); @@ -195,6 +196,10 @@ long maxDeltaStart() { return committedMaxDeltaStart; } + long maxSchemaFsn() { + return maxSchemaFsn; + } + long rawAddr() { return rawAddr; } @@ -211,10 +216,6 @@ int rawCapacity() { return rawCapacity; } - boolean requiresSchema() { - return requiresSchema; - } - /** * Releases the cached suffix after a foreground producer and its one send * loop have both consumed it. Recovery metadata remains available, but the diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index 998b6e585..046eba3b6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -25,9 +25,11 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; @@ -789,6 +791,29 @@ public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception }); } + @Test + public void testSchemaRequirementTracksUnackedSchemaFrames() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 1 << 20); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer rows = schemaTestRows()) { + assertEquals(0L, appendLegacy(engine, encoder, rows)); + assertFalse(engine.requiresSchema()); + assertEquals(1L, appendSchema(engine, encoder, rows)); + assertTrue(engine.requiresSchema()); + assertEquals(2L, appendLegacy(engine, encoder, rows)); + assertTrue(engine.acknowledge(0L)); + assertTrue("the schema frame is still unacked", engine.requiresSchema()); + assertTrue(engine.acknowledge(1L)); + assertFalse(engine.requiresSchema()); + assertEquals(3L, appendSchema(engine, encoder, rows)); + assertTrue(engine.requiresSchema()); + assertTrue(engine.acknowledge(3L)); + assertFalse(engine.requiresSchema()); + } + }); + } + @Test public void testSealedSegmentsAccumulateAfterRotation() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -1038,6 +1063,14 @@ private static Thread workerThread(SegmentManager manager) throws Exception { // a bounded wait. The probe is a second mapping of the same file the manager // worker writes through; its MAP_SHARED reads observe the worker's writes, and // it is closed before the next session opens the slot. + private static long appendLegacy(CursorSendEngine engine, QwpWebSocketEncoder encoder, QwpTableBuffer rows) { + return engine.appendBlocking(encoder.getBuffer().getBufferPtr(), encoder.encode(rows)); + } + + private static long appendSchema(CursorSendEngine engine, QwpWebSocketEncoder encoder, QwpTableBuffer rows) { + return engine.appendBlocking(encoder.getBuffer().getBufferPtr(), encoder.encodeSchema(rows, -1, -1)); + } + private static void awaitManagerPersistedWatermark(String slotDir, long expected) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); long last = AckWatermark.INVALID; @@ -1055,4 +1088,11 @@ private static void awaitManagerPersistedWatermark(String slotDir, long expected + " within 10s (last on-disk read=" + last + ")"); } + private static QwpTableBuffer schemaTestRows() { + QwpTableBuffer rows = new QwpTableBuffer("schema_requirement"); + rows.getOrCreateColumn("n", QwpConstants.TYPE_LONG, true).addLong(1); + rows.nextRow(); + return rows; + } + } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java index 511fbbcf7..9d928f9cb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -572,7 +572,7 @@ public WebSocketClient reconnect() { } @Override - public void requireSchema() { + public void setSchemaRequired(boolean isRequired) { } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java index 4397e3b4b..a8df48295 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -55,12 +55,46 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; public class QwpSchemaReplayNetworkTest { private static final int SEGMENT_SIZE = 1 << 20; @Rule public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + @Test + public void testBackgroundDrainerRetiresOrphanSchemaTailWithoutConnecting() throws Exception { + TestUtils.assertMemoryLeak(() -> { + File root = temp.newFolder("drainer-orphan-schema-tail"); + String slot = new File(root, "ghost").getAbsolutePath(); + seedDeferredSchemaTail(slot); + RecordingHandler oldHandler = new RecordingHandler(false, "deferred"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + IgnoringFactory factory = new IgnoringFactory(oldPeer.getPort(), 1); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, SEGMENT_SIZE, 4L * SEGMENT_SIZE, factory, + 5_000, 1, 4, false, 0 + ); + Thread thread = new Thread(drainer::run, "schema-orphan-tail-drainer"); + thread.start(); + try { + thread.join(5_000); + Assert.assertFalse("drainer did not finish", thread.isAlive()); + } finally { + drainer.requestStop(); + thread.join(5_000); + } + // The aborted tail is retired locally, so the drainer never needs a peer. + Assert.assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + Assert.assertEquals(0, factory.attempts.get()); + Assert.assertEquals(0, oldHandler.frames.size()); + } + assertNoQuarantine(root); + }); + } + @Test public void testBackgroundDrainerRejectsUnconfirmedClientReturnedByFactoryWithoutQuarantine() throws Exception { File root = temp.newFolder("drainer-old-peer"); @@ -95,6 +129,58 @@ public void testBackgroundDrainerRejectsUnconfirmedClientReturnedByFactoryWithou assertNoQuarantine(root); } + @Test(timeout = 20_000) + public void testAckedSchemaFramesDoNotBlockLegacyPeerAfterDowngrade() throws Exception { + TestUtils.assertMemoryLeak(() -> { + File root = temp.newFolder("acked-then-downgrade"); + String slot = new File(root, "default").getAbsolutePath(); + RecordingHandler supportingHandler = new RecordingHandler(true, "downgrade"); + RecordingHandler legacyHandler = new RecordingHandler(true, "downgrade"); + try (TestWebSocketServer legacyPeer = new TestWebSocketServer(legacyHandler); + CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer schemaRows = table("downgrade", 1); + QwpTableBuffer legacyRows = table("downgrade", 2)) { + legacyPeer.start(); + Assert.assertTrue(legacyPeer.awaitStart(5, TimeUnit.SECONDS)); + SchemaAwareFactory factory = new SchemaAwareFactory(legacyPeer.getPort(), 1); + CursorWebSocketSendLoop loop = null; + try { + try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { + supportingPeer.setAdvertiseSchema(true); + supportingPeer.start(); + Assert.assertTrue(supportingPeer.awaitStart(5, TimeUnit.SECONDS)); + WebSocketClient initialClient = connectLegacyClient(supportingPeer.getPort(), true); + loop = new CursorWebSocketSendLoop( + initialClient, engine, 0, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, 1, 4, false); + Assert.assertTrue(initialClient.isQwpSchemaEnabled()); + loop.start(); + int length = encoder.encodeSchema(schemaRows, -1, -1); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + Assert.assertTrue(engine.requiresSchema()); + awaitAckedFsn(engine, 0); + Assert.assertFalse("an acked schema frame must not pin the capability", + engine.requiresSchema()); + } + // The schema-capable peer is gone; the loop must accept the legacy peer. + int length = encoder.encode(legacyRows); + Assert.assertEquals(1, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + awaitAckedFsn(engine, 1); + Assert.assertNull(loop.getTerminalError()); + Assert.assertEquals(1, supportingHandler.frames.size()); + Assert.assertEquals(1, legacyHandler.frames.size()); + Assert.assertEquals(0, legacyHandler.frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA); + } finally { + if (loop != null) { + loop.close(); + } + } + } + }); + } + @Test public void testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames() throws Exception { File root = temp.newFolder("mixed"); @@ -395,7 +481,7 @@ private static void assertSchemaRequirementIsolated(int port, boolean foreground if (foregroundFirst) { foreground = sender.newReconnectFactory(); background = sender.newBackgroundReconnectFactory(() -> false); - foreground.requireSchema(); + foreground.setSchemaRequired(true); assertSchemaMismatch(foreground); try (WebSocketClient client = background.reconnect()) { Assert.assertFalse(client.isQwpSchemaEnabled()); @@ -403,7 +489,7 @@ private static void assertSchemaRequirementIsolated(int port, boolean foreground } else { background = sender.newBackgroundReconnectFactory(() -> false); foreground = sender.newReconnectFactory(); - background.requireSchema(); + background.setSchemaRequired(true); assertSchemaMismatch(background); try (WebSocketClient client = foreground.reconnect()) { Assert.assertFalse(client.isQwpSchemaEnabled()); @@ -426,6 +512,16 @@ private static void assertSchemaMismatch(CursorWebSocketSendLoop.ReconnectFactor } } + private static void awaitAckedFsn(CursorSendEngine engine, long fsn) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (engine.ackedFsn() < fsn) { + if (System.nanoTime() > deadline) { + Assert.fail("timed out waiting for ackedFsn " + fsn + ", actual " + engine.ackedFsn()); + } + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + } + private static String config(int port, File root, boolean durableAck) { return "ws::addr=localhost:" + port + ";sf_dir=" + root.getAbsolutePath() @@ -627,7 +723,7 @@ public WebSocketClient reconnect() { } @Override - public void requireSchema() { + public void setSchemaRequired(boolean isRequired) { // Deliberately ignored: returned-client validation must still reject it. } } @@ -705,7 +801,7 @@ private static byte[] schemaFeedbackInvalidColumns(long sequence) { private static final class SchemaAwareFactory implements CursorWebSocketSendLoop.ReconnectFactory { private final CountDownLatch attemptsLatch; private final int port; - private volatile boolean schemaRequired; + private volatile boolean isSchemaRequired; private SchemaAwareFactory(int port, int expectedAttempts) { this.port = port; @@ -719,12 +815,12 @@ boolean awaitAttempts(long timeout, TimeUnit unit) throws InterruptedException { @Override public WebSocketClient reconnect() { attemptsLatch.countDown(); - return connectLegacyClient(port, schemaRequired); + return connectLegacyClient(port, isSchemaRequired); } @Override - public void requireSchema() { - schemaRequired = true; + public void setSchemaRequired(boolean isRequired) { + isSchemaRequired = isRequired; } } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java index 5f09b51fa..ce61cb122 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java @@ -126,7 +126,7 @@ public void testAcknowledgedExtendedResidueDoesNotRequireSchema() throws Excepti } @Test - public void testExtendedDeferredTailCannotRetireBeforeConfirmation() throws Exception { + public void testExtendedDeferredTailRetiresLocallyAndDropsSchemaRequirement() throws Exception { assertMemoryLeak(() -> { Path slot = temporaryFolder.newFolder("qwp-schema-tail").toPath(); long payload = qwpFrame((byte) (QwpConstants.FLAG_SCHEMA | QwpConstants.FLAG_DEFER_COMMIT)); @@ -136,10 +136,10 @@ public void testExtendedDeferredTailCannotRetireBeforeConfirmation() throws Exce } try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { Assert.assertTrue(recovered.requiresSchema()); - Assert.assertFalse(recovered.retireRecoveredOrphanTailIfReady()); - Assert.assertEquals(-1, recovered.ackedFsn()); - Assert.assertTrue(recovered.retireRecoveredOrphanTailIfReady(true)); + // Retirement transmits nothing, so it needs no schema-capable peer. + Assert.assertTrue(recovered.retireRecoveredOrphanTailIfReady()); Assert.assertEquals(0, recovered.ackedFsn()); + Assert.assertFalse(recovered.requiresSchema()); } } finally { Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); From c314537af6dd9957a64ce6eb97c6e21a6b0dfed3 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 13:05:20 +0200 Subject: [PATCH 37/51] Resolve schema columns once per binding QwpSchemaBinding re-validated and re-hashed the column name on every value: isValidColumnName, a schema hash lookup, and the buffer lookup, plus the designated, wire-type and parameterized checks. All of these depend only on the column, so every row paid for work whose answer never changes while the binding is attached. QwpTableBuffer.ColumnBuffer now records the schema index the binding resolved it to (-1 when the schema lacks the column). Each write first finds the column through the buffer's sequential cursor; a resolved column skips name validation, the schema lookup and the per-column checks, leaving one name compare plus the value conversion, the same lookup cost as the legacy path. A binding only attaches to an empty buffer and unbinding closes its columns, so a recorded resolution can never outlive its schema. Columns without one fall back to the full path. The parameterized-type check now runs before column creation, so a rejected write no longer leaves an empty column behind. The three copies of the inferred-column logic share isInferredWritable and claimInferred. QwpSchemaBindingBenchmark (10 columns, 5 LONG + 5 DOUBLE) measures the schema path at 42 ns/row against 31 ns/row for legacy, down from 141 ns (2-char names) and 282 ns (14-char names). The remaining ~1 ns per value is the target-type dispatch. --- .../qwp/protocol/QwpSchemaBinding.java | 224 ++++++++++++------ .../cutlass/qwp/protocol/QwpTableBuffer.java | 74 ++++-- .../protocol/QwpSchemaBindingBenchmark.java | 127 ++++++++++ .../QwpSchemaBindingColumnResolutionTest.java | 155 ++++++++++++ 4 files changed, 479 insertions(+), 101 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index f1bff987b..921a5b6ee 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -151,28 +151,21 @@ public QwpSchemaBinding decimalColumn(CharSequence name, CharSequence value, Dec if (value == null || value.length() == 0) { return this; } - int index = targetIndex(name, "DECIMAL256"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DECIMAL256"); int targetType = targetType(index, ColumnType.DECIMAL256); if (index < 0) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(name); - if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + if (!isInferredWritable(existing, name, "DECIMAL256", QwpConstants.TYPE_DECIMAL256)) { return this; } - if (inferred != null && inferred.getType() != QwpConstants.TYPE_DECIMAL256) { - throw unsupported(name, "DECIMAL256", -1, "inferred column type conflict [inferredType=" - + QwpConstants.getTypeName(inferred.getType()) + ']'); - } parseDecimalText(name, value, scratch, targetType); - inferred = buffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); - if (inferred != null) { - // Keep the legacy public-overload contract: infer DECIMAL256 at - // the parsed natural scale. A parsed special is an effective - // null but does not pin the scale of a later finite value. - inferred.addDecimal256(scratch); - } + // Keep the legacy public-overload contract: infer DECIMAL256 at + // the parsed natural scale. A parsed special is an effective + // null but does not pin the scale of a later finite value. + claimInferred(existing, name, QwpConstants.TYPE_DECIMAL256).addDecimal256(scratch); return this; } - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "DECIMAL256", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "DECIMAL256", index, targetType); if (column == null) { return this; } @@ -222,9 +215,10 @@ public QwpSchemaBinding binaryColumn(CharSequence name, long ptr, long len) { public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "BOOLEAN"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "BOOLEAN"); int targetType = targetType(index, ColumnType.BOOLEAN); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BOOLEAN", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "BOOLEAN", index, targetType); if (column == null) { return this; } @@ -301,12 +295,12 @@ public QwpSchemaBinding doubleArray(CharSequence name, DoubleArray array) { if (array == null) { return this; } - int index = targetIndex(name, "DOUBLE_ARRAY"); - QwpTableBuffer.ColumnBuffer existing = buffer.getExistingColumnByName(name); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DOUBLE_ARRAY"); if (existing != null && existing.getSize() > buffer.getRowCount()) { return this; } - QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, index, array.getDimensionality()); + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(existing, name, index, array.getDimensionality()); if (column != null) { column.addDoubleArray(array); } @@ -389,9 +383,10 @@ public QwpSchemaBinding geoHashColumn(CharSequence name, long bits, int precisio "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); } int sourceType = ColumnType.getGeoHashTypeWithBits(precisionBits); - int index = targetIndex(name, "GEOHASH"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "GEOHASH"); int targetType = targetType(index, sourceType); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "GEOHASH", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "GEOHASH", index, targetType); if (column == null) { return this; } @@ -440,9 +435,10 @@ public QwpSchemaBinding intColumn(CharSequence name, int value) { public QwpSchemaBinding ipv4Column(CharSequence name, int address) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "IPv4"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "IPv4"); int targetType = targetType(index, ColumnType.IPv4); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "IPv4", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "IPv4", index, targetType); if (column == null) { return this; } @@ -466,9 +462,10 @@ public QwpSchemaBinding ipv4Column(CharSequence name, CharSequence address) { if (address == null) { return this; } - int index = targetIndex(name, "IPv4"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "IPv4"); int targetType = targetType(index, ColumnType.IPv4); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "IPv4", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "IPv4", index, targetType); if (column == null) { return this; } @@ -499,9 +496,10 @@ public QwpSchemaBinding ipv4Column(CharSequence name, CharSequence address) { public QwpSchemaBinding longColumn(CharSequence name, long value) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "LONG"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "LONG"); int targetType = targetType(index, ColumnType.LONG); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "LONG", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "LONG", index, targetType); if (column == null) { return this; } @@ -567,9 +565,10 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { public QwpSchemaBinding long256Column(CharSequence name, long l0, long l1, long l2, long l3) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "LONG256"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "LONG256"); int targetType = targetType(index, ColumnType.LONG256); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "LONG256", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "LONG256", index, targetType); if (column == null) { return this; } @@ -595,9 +594,10 @@ public QwpSchemaBinding shortColumn(CharSequence name, short value) { public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "STRING"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "STRING"); int targetType = targetType(index, ColumnType.VARCHAR); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "STRING", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "STRING", index, targetType); if (column == null) { return this; } @@ -696,9 +696,10 @@ public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { public QwpSchemaBinding charColumn(CharSequence name, char value) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "CHAR"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "CHAR"); int targetType = targetType(index, ColumnType.CHAR); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "CHAR", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "CHAR", index, targetType); if (column == null) { return this; } @@ -728,9 +729,10 @@ public QwpSchemaBinding charColumn(CharSequence name, char value) { public QwpSchemaBinding symbol(CharSequence name, CharSequence value) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "SYMBOL"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "SYMBOL"); int targetType = targetType(index, ColumnType.SYMBOL); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "SYMBOL", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "SYMBOL", index, targetType); if (column == null) { return this; } @@ -757,10 +759,11 @@ public QwpSchemaBinding symbol(CharSequence name, CharSequence value) { */ public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUnit unit) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "TIMESTAMP"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "TIMESTAMP"); int targetType = targetType(index, unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "TIMESTAMP", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "TIMESTAMP", index, targetType); if (column == null) { return this; } @@ -801,9 +804,10 @@ public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUni */ public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "TIMESTAMP"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "TIMESTAMP"); int targetType = targetType(index, ColumnType.TIMESTAMP_MICRO); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "TIMESTAMP", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "TIMESTAMP", index, targetType); if (column == null) { return this; } @@ -834,9 +838,10 @@ public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, "UUID"); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "UUID"); int targetType = targetType(index, ColumnType.UUID); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "UUID", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "UUID", index, targetType); if (column == null) { return this; } @@ -858,8 +863,9 @@ public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { /** Rejects an effective input family whose schema-mode conversion is not implemented. */ public QwpSchemaBinding unsupportedColumn(CharSequence name, String inputType) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, inputType); - QwpTableBuffer.ColumnBuffer inferred = index < 0 ? buffer.getExistingColumnByName(name) : null; + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + QwpTableBuffer.ColumnBuffer inferred = index < 0 ? existing : null; if (inferred != null && inferred.getSize() > buffer.getRowCount()) { return this; } @@ -867,7 +873,7 @@ public QwpSchemaBinding unsupportedColumn(CharSequence name, String inputType) { throw unsupported(name, inputType, -1, "input conversion is not implemented"); } int targetType = schema.getColumnType(index); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); if (column != null) { throw unsupported(name, inputType, targetType, "input conversion is not implemented"); } @@ -941,10 +947,56 @@ private void requireUsableSchema() { } } + /** Claims or creates the column for a name the schema lacks; call after {@link #isInferredWritable}. */ + private QwpTableBuffer.ColumnBuffer claimInferred( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + byte wireType + ) { + QwpTableBuffer.ColumnBuffer column = existing != null + ? buffer.claimColumn(existing) + : buffer.getOrCreateColumn(name, wireType, true); + column.setSchemaIndex(-1); + return column; + } + + /** + * Finds the buffer column for {@code name}, advancing the buffer's + * sequential cursor. An invalid name never matches, so the caller still + * rejects it through {@link #targetIndex}. + */ + private QwpTableBuffer.ColumnBuffer existingColumn(CharSequence name) { + return name == null || name.length() == 0 ? null : buffer.findColumn(name); + } + + /** + * Returns false for a duplicate write to a column the schema lacks, and + * rejects a write whose wire type conflicts with the inferred column. + */ + private boolean isInferredWritable( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + String inputType, + byte wireType + ) { + if (existing == null) { + return true; + } + if (existing.getSize() > buffer.getRowCount()) { + return false; + } + if (existing.getType() != wireType) { + throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" + + QwpConstants.getTypeName(existing.getType()) + ']'); + } + return true; + } + private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String inputType, int expectedTargetType) { - int index = targetIndex(name, inputType); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); int targetType = targetType(index, expectedTargetType); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); if (column != null && targetType != expectedTargetType) { throw unsupported(name, inputType, targetType, "conversion is not implemented"); } @@ -952,17 +1004,19 @@ private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String input } private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn(CharSequence name, int sourceDimensions) { - int index = targetIndex(name, "DOUBLE_ARRAY"); - return targetDoubleArrayColumn(name, index, sourceDimensions); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DOUBLE_ARRAY"); + return targetDoubleArrayColumn(existing, name, index, sourceDimensions); } private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn( + QwpTableBuffer.ColumnBuffer existing, CharSequence name, int index, int sourceDimensions ) { int targetType = targetType(index, ColumnType.encodeArrayType(ColumnType.DOUBLE, sourceDimensions)); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, "DOUBLE_ARRAY", index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "DOUBLE_ARRAY", index, targetType); if (column == null) { return null; } @@ -991,36 +1045,47 @@ private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn( return column; } + /** + * Returns the column {@code name} writes to for the in-progress row, or + * {@code null} for a duplicate write. {@code existing} is the buffer's + * column for {@code name}, found by {@link #existingColumn}. + */ private QwpTableBuffer.ColumnBuffer targetColumn( + QwpTableBuffer.ColumnBuffer existing, CharSequence name, String inputType, int index, int targetType ) { if (index < 0) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(name); - if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + final byte wireType = wireType(targetType); + if (!isInferredWritable(existing, name, inputType, wireType)) { return null; } - if (inferred != null && inferred.getType() != wireType(targetType)) { - throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" - + QwpConstants.getTypeName(inferred.getType()) + ']'); + if (wireType == 0) { + throw unsupported(name, inputType, targetType, "conversion is not implemented"); } + return claimInferred(existing, name, wireType); + } + if (existing != null && existing.isSchemaResolved()) { + // Every check below depends only on the schema column, and passed + // when this binding created the column. + return buffer.claimColumn(existing); } - if (index >= 0 && index == schema.getDesignatedIndex()) { + if (index == schema.getDesignatedIndex()) { throw unsupported(name, inputType, targetType, "designated timestamp writes are not implemented"); } final byte wireType = wireType(targetType); if (wireType == 0) { throw unsupported(name, inputType, targetType, "conversion is not implemented"); } - QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn(name, wireType, true); - if (column == null) { - return null; - } - if (index >= 0 && schema.hasColumnExtensionParameters(index)) { + if (schema.hasColumnExtensionParameters(index)) { throw unsupported(name, inputType, targetType, "parameterized target type"); } + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn(name, wireType, true); + if (column != null) { + column.setSchemaIndex(index); + } return column; } @@ -1124,20 +1189,13 @@ private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String if (value == null || value.isNull()) { return this; } - int index = targetIndex(name, inputType); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); if (index < 0) { - QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(name); - if (inferred != null && inferred.getSize() > buffer.getRowCount()) { - return this; - } - if (inferred != null && inferred.getType() != inferredWireType) { - throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" - + QwpConstants.getTypeName(inferred.getType()) + ']'); - } - inferred = buffer.getOrCreateColumn(name, inferredWireType, true); - if (inferred == null) { + if (!isInferredWritable(existing, name, inputType, inferredWireType)) { return this; } + QwpTableBuffer.ColumnBuffer inferred = claimInferred(existing, name, inferredWireType); if (value instanceof Decimal64) { inferred.addDecimal64((Decimal64) value); } else if (value instanceof Decimal128) { @@ -1148,7 +1206,7 @@ private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String return this; } int targetType = targetType(index, ColumnType.DECIMAL256); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); if (column == null) { return this; } @@ -1224,9 +1282,10 @@ private QwpSchemaBinding integerColumn( boolean sourceNull ) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, inputType); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); int targetType = targetType(index, inferredType); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); if (column == null) { return this; } @@ -1666,10 +1725,11 @@ private static short parseSchemaShort(CharSequence value) throws NumericExceptio private QwpSchemaBinding floatingNumericColumn(CharSequence name, double value, String inputType) { buffer.requireSchemaBinding(this); - int index = targetIndex(name, inputType); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); int targetType = targetType(index, "FLOAT".equals(inputType) ? ColumnType.FLOAT : ColumnType.DOUBLE); - QwpTableBuffer.ColumnBuffer column = targetColumn(name, inputType, index, targetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); if (column == null) { return this; } @@ -1837,7 +1897,15 @@ private byte timestampWireType(int targetType) { throw unsupported(null, "TIMESTAMP", targetType, "designated timestamp conversion is not implemented"); } - private int targetIndex(CharSequence name, String inputType) { + /** + * Returns the schema index of {@code name}, or -1 when the schema lacks it. + * A column this binding already resolved answers without re-validating or + * re-hashing the name: it can only match a name equal to its own valid one. + */ + private int targetIndex(QwpTableBuffer.ColumnBuffer existing, CharSequence name, String inputType) { + if (existing != null && existing.isSchemaResolved()) { + return existing.getSchemaIndex(); + } if (name == null || !TableUtils.isValidColumnName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { throw error(INVALID_VALUE, name, inputType, null, "invalid column name"); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 398a02136..51fee5afb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -198,6 +198,37 @@ ColumnBuffer getExistingColumnByName(CharSequence name) { return index >= 0 ? columns.get(index) : null; } + /** + * Claims {@code column}, found by {@link #findColumn}, for the in-progress + * row. Returns {@code null} when the row already holds a value for it, with + * the same first-value-wins semantics as {@link #getOrCreateColumn}. + */ + ColumnBuffer claimColumn(ColumnBuffer column) { + if (column.size > rowCount) { + return null; + } + inProgressColumnCount++; + return column; + } + + /** + * Finds a column by name without checking its type, advancing the + * sequential access cursor on a predicted hit exactly like + * {@link #getOrCreateColumn}. Returns {@code null} when absent. + */ + ColumnBuffer findColumn(CharSequence name) { + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (Chars.equalsIgnoreCase(candidate.name, name)) { + columnAccessCursor++; + return candidate; + } + } + int idx = columnNameToIndex.get(name); + return idx >= 0 ? columns.get(idx) : null; + } + /** * Gets or creates a column with the given name and type. *

      @@ -219,11 +250,7 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean useN // col.size > rowCount means this column already received a value // for the in-progress row. Silently ignore the duplicate (first // value wins, same as the ILP server behaviour). - if (existing.size > rowCount) { - return null; - } - inProgressColumnCount++; - return existing; + return claimColumn(existing); } if (TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { ColumnBuffer col = createColumn(name, type, useNullBitmap); @@ -457,26 +484,11 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullB } private ColumnBuffer lookupColumn(CharSequence name, byte type) { - // Fast path: predict next column in sequence - int n = columns.size(); - if (columnAccessCursor < n) { - ColumnBuffer candidate = fastColumns[columnAccessCursor]; - if (Chars.equalsIgnoreCase(candidate.name, name)) { - columnAccessCursor++; - assertColumnType(name, type, candidate); - return candidate; - } - } - - // Slow path: hash map lookup - int idx = columnNameToIndex.get(name); - if (idx >= 0) { - ColumnBuffer existing = columns.get(idx); + ColumnBuffer existing = findColumn(name); + if (existing != null) { assertColumnType(name, type, existing); - return existing; } - - return null; + return existing; } private void rebuildColumnAccessStructures() { @@ -645,6 +657,7 @@ void reset(boolean forLong) { * operation and efficient bulk copy to network buffers. */ public static class ColumnBuffer implements QuietCloseable { + static final int SCHEMA_UNRESOLVED = Integer.MIN_VALUE; private static final long DOUBLE_ARRAY_BASE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(double[].class); final int elemSize; final String name; @@ -664,6 +677,9 @@ public static class ColumnBuffer implements QuietCloseable { // Decimal storage private byte decimalScale = -1; private boolean schemaDecimalScaleLocked; + // Schema column index the binding resolved this column to, -1 when the + // schema lacks it, or SCHEMA_UNRESOLVED before any binding resolved it. + private int schemaIndex = SCHEMA_UNRESOLVED; private int inferredArrayDimensionality = -1; private double[] doubleArrayData; // GeoHash precision (number of bits, 1-60) @@ -1531,10 +1547,18 @@ public long getNullBitmapAddress() { return nullBufPtr; } + int getSchemaIndex() { + return schemaIndex; + } + public int getSize() { return size; } + void setSchemaIndex(int schemaIndex) { + this.schemaIndex = schemaIndex; + } + int pinInferredArrayDimensionality(int dimensionality) { if (inferredArrayDimensionality < 0) { inferredArrayDimensionality = dimensionality; @@ -1588,6 +1612,10 @@ public boolean hasSymbol(CharSequence value) { return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; } + boolean isSchemaResolved() { + return schemaIndex != SCHEMA_UNRESOLVED; + } + public boolean isNull(int index) { if (nullBufPtr == 0 || index >= nullBufCapRows) { return false; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java new file mode 100644 index 000000000..beaf1de4d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java @@ -0,0 +1,127 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +/** + * Per-row cost of writing an exact-type row through {@link QwpSchemaBinding} + * versus the legacy {@link QwpTableBuffer#getOrCreateColumn} path. Both write + * the same 10 columns (5 LONG, 5 DOUBLE) in the same order, so the difference + * is the binding's per-value column resolution. Run with {@link #main}. + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 2) +@Fork(1) +public class QwpSchemaBindingBenchmark { + private static final int COLUMN_COUNT = 10; + private static final int ROWS_PER_BATCH = 1_000; + private final String[] names = new String[COLUMN_COUNT]; + private QwpSchemaBinding binding; + private QwpTableBuffer legacyBuffer; + @Param({"2", "14"}) + private int nameLength; + private QwpTableBuffer schemaBuffer; + private long value; + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(QwpSchemaBindingBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void legacyRow() { + long v = value++; + for (int i = 0; i < COLUMN_COUNT; i += 2) { + legacyBuffer.getOrCreateColumn(names[i], QwpConstants.TYPE_LONG, true).addLong(v); + legacyBuffer.getOrCreateColumn(names[i + 1], QwpConstants.TYPE_DOUBLE, true).addDouble(v); + } + legacyBuffer.nextRow(); + if (legacyBuffer.getRowCount() == ROWS_PER_BATCH) { + legacyBuffer.reset(); + } + } + + @Benchmark + public void schemaRow() { + long v = value++; + for (int i = 0; i < COLUMN_COUNT; i += 2) { + binding.longColumn(names[i], v); + binding.doubleColumn(names[i + 1], v); + } + schemaBuffer.nextRow(); + if (schemaBuffer.getRowCount() == ROWS_PER_BATCH) { + schemaBuffer.reset(); + } + } + + @Setup(Level.Trial) + public void setUp() { + byte[][] columns = new byte[COLUMN_COUNT][]; + for (int i = 0; i < COLUMN_COUNT; i++) { + StringBuilder name = new StringBuilder().append((char) ('a' + i)); + while (name.length() < nameLength) { + name.append('x'); + } + names[i] = name.toString(); + columns[i] = QwpSchemaTestFixtures.column(names[i], i % 2 == 0 ? ColumnType.LONG : ColumnType.DOUBLE); + } + legacyBuffer = new QwpTableBuffer("t"); + schemaBuffer = new QwpTableBuffer("t"); + binding = QwpSchemaTestFixtures.binding(schemaBuffer, columns); + } + + @TearDown(Level.Trial) + public void tearDown() { + legacyBuffer.close(); + schemaBuffer.close(); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java new file mode 100644 index 000000000..4afa3669a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java @@ -0,0 +1,155 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * A binding resolves each column against the schema once, when it creates the + * buffer column, and later writes reuse that resolution. These tests pin that + * the reuse keeps every per-value rule of the first write. + */ +public class QwpSchemaBindingColumnResolutionTest { + + @Test + public void testDuplicateWriteToResolvedColumnKeepsFirstValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 1); + buffer.nextRow(); + binding.longColumn("v", 2); + binding.longColumn("V", 3); + buffer.nextRow(); + assertInts(buffer.getColumn(0), 1, 2); + } + }); + } + + @Test + public void testInferredColumnTypeConflictIsDetectedOnLaterRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("x", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleColumn("x", 2.0), "inferred column type conflict"); + buffer.cancelCurrentRow(); + binding.longColumn("X", 3); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + QwpTableBuffer.ColumnBuffer column = buffer.getColumn(0); + Assert.assertEquals(QwpConstants.TYPE_LONG, column.getType()); + Assert.assertEquals(2, column.getSize()); + Assert.assertEquals(1, Unsafe.getUnsafe().getLong(column.getDataAddress())); + Assert.assertEquals(3, Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES)); + } + }); + } + + @Test + public void testInvalidNameIsRejectedAfterColumnsExist() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("", 2), "invalid column name"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn(null, 2), "invalid column name"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("v?", 2), "invalid column name"); + Assert.assertEquals(1, buffer.getColumnCount()); + } + }); + } + + @Test + public void testRejectedSchemaColumnsStayRejectedAcrossRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, + column("ts", ColumnType.TIMESTAMP_MICRO), + parameterizedColumn("p", ColumnType.LONG), + column("v", ColumnType.INT)); + for (int attempt = 0; attempt < 2; attempt++) { + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("ts", 1), "designated timestamp writes"); + buffer.cancelCurrentRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("p", 1), "parameterized target type"); + buffer.cancelCurrentRow(); + binding.longColumn("v", attempt); + buffer.nextRow(); + } + Assert.assertEquals("rejected targets must not create buffer columns", 1, buffer.getColumnCount()); + assertInts(buffer.getColumn(0), 0, 1); + } + }); + } + + @Test + public void testResolvedColumnKeepsConversionAndRangeChecks() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 7); + buffer.nextRow(); + binding.longColumn("V", 8); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("v", 1L << 40)); + buffer.cancelCurrentRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.binaryColumn("v", new byte[]{1}), "conversion is not implemented"); + buffer.cancelCurrentRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + Assert.assertEquals(QwpConstants.TYPE_INT, buffer.getColumn(0).getType()); + assertInts(buffer.getColumn(0), 7, 8); + } + }); + } + + private static void assertInts(QwpTableBuffer.ColumnBuffer column, int... expected) { + Assert.assertEquals(expected.length, column.getSize()); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals(expected[i], Unsafe.getUnsafe().getInt(column.getDataAddress() + (long) i * Integer.BYTES)); + } + } +} From e8b1c5b3f5b4abf369fab0577e105e54d73b0c27 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 13:40:01 +0200 Subject: [PATCH 38/51] Parse schema timestamp text in a single pass QwpSchemaTimestampParser did work whose outcome was already decided. Every ISO timestamp first ran the PG parser to completion of the day and failed there. parse() fetched an exception on the success path. Each fraction attempt copied a Parsed object, the greedy fraction threw to signal a miss, numeric zones allocated a Zone, and named zones were found by a linear scan over ~4400 names with an iterator per value. The first timestamp with any zone, even 'Z', loaded every zone's rules on the producer thread. appendStringTimestamp also UTF-8 encoded each value into a sink before parsing it back through a byte view. The character after the day now selects the parser: only the PG layout has a space there, and a space fails every ISO layout. DATE dispatches the same way between PG, UTC and a raw number. Fields live in locals, the greedy fraction returns a sentinel, numeric zones return an offset, and named zones sit in a hash table inside a holder class that 'Z' and numeric offsets never initialize. TIMESTAMP text now parses the caller's CharSequence directly. The server parses TIMESTAMP text as UTF-8 bytes, where no non-ASCII byte matches anything, so named-zone lookup rejects non-ASCII for TIMESTAMP; DATE keeps its UTF-16 matching, as the server's fallback does. Zone names are upper-cased with Locale.ROOT instead of the default locale. Under a Turkish default locale about 2240 names containing 'i' could never match; results elsewhere are unchanged. The client corpora, the OSS e2e suites against a real server (22 tests, including named-zone gap, overlap, alias and history cases) and 2471 QWP client tests pass; the binding tests also pass on JDK 8. --- .../qwp/protocol/QwpSchemaBinding.java | 9 +- .../protocol/QwpSchemaTimestampParser.java | 499 ++++++++++-------- 2 files changed, 286 insertions(+), 222 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index 921a5b6ee..d40aed680 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -1598,15 +1598,8 @@ private void appendStringTimestamp( CharSequence value, int targetType ) { - Utf8StringSink sink = timestampTextSink; - if (sink == null) { - timestampTextSink = sink = new Utf8StringSink(Math.max(32, value.length())); - } else { - sink.clear(); - } - sink.put(value); try { - long micros = QwpSchemaTimestampParser.parse(sink.asAsciiCharSequence()); + long micros = QwpSchemaTimestampParser.parse(value); if (targetType == ColumnType.TIMESTAMP_NANO) { if (micros > Long.MAX_VALUE / 1000 || micros < Long.MIN_VALUE / 1000) { throw NumericException.instance(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java index 0e91217ea..03be1d108 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java @@ -17,13 +17,10 @@ import java.time.zone.ZoneOffsetTransitionRule; import java.time.zone.ZoneRules; import java.time.zone.ZoneRulesProvider; -import java.util.ArrayList; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; /** * Source-faithful parser for the fixed formats accepted by QWP STRING-to-timestamp @@ -33,6 +30,7 @@ */ final class QwpSchemaTimestampParser { private static final long DAY = 86_400_000_000L; + private static final long GREEDY_MILLIS_FAILED = -1L; private static final long DATE_DAY = 86_400_000L; private static final long DATE_HOUR = 3_600_000L; private static final long DATE_MINUTE = 60_000L; @@ -51,42 +49,46 @@ final class QwpSchemaTimestampParser { private QwpSchemaTimestampParser() { } + /** + * Parses TIMESTAMP text to epoch micros. The server parses the UTF-8 bytes, + * where no non-ASCII byte matches anything, so any non-ASCII character fails. + */ static long parse(CharSequence value) throws NumericException { - NumericException failure = NumericException.instance(); - try { - return parsePg(value); - } catch (NumericException ignored) { - } - try { - return parseIso(value); - } catch (NumericException ignored) { + // Only the PG layout has a space after the day, and a space there fails + // every ISO layout, so that one character selects the parser. + int n = value.length(); + int dash = firstDash(value, n); + if (dash >= 1 && dash + 6 < n && value.charAt(dash + 6) == ' ') { + return parsePg(value, dash); } - throw failure; + return parseIso(value); } + /** Parses DATE text to epoch millis; the server parses DATE text as UTF-16. */ static long parseDate(CharSequence value) throws NumericException { - NumericException failure = NumericException.instance(); - try { - return parsePgDate(value); - } catch (NumericException ignored) { + int n = value.length(); + int dash = firstDash(value, n); + if (dash < 1) { + // Both date layouts need an inner dash and a raw number cannot hold one. + return Numbers.parseLong(value); } - try { - return parseUtcDate(value); - } catch (NumericException ignored) { + // PG ends at the day or continues with a space; only UTC has a 'T' there. + int separator = dash + 6; + if (separator >= n || value.charAt(separator) == ' ') { + return parsePgDate(value, dash); } - try { - return Numbers.parseLong(value); - } catch (NumericException ignored) { + if (value.charAt(separator) == 'T') { + return parseUtcDate(value); } - throw failure; + throw NumericException.instance(); } - private static long parsePgDate(CharSequence value) throws NumericException { + private static int firstDash(CharSequence value, int n) { + return indexOf(value, '-', n > 0 && value.charAt(0) == '-' ? 1 : 0, n); + } + + private static long parsePgDate(CharSequence value, int dash) throws NumericException { int n = value.length(); - int dash = indexOf(value, '-', n > 0 && value.charAt(0) == '-' ? 1 : 0, n); - if (dash < 1) { - throw NumericException.instance(); - } int year = greedyInt(value, 0, dash); if (dash == 2) { year += REFERENCE_CENTURY; @@ -96,7 +98,7 @@ private static long parsePgDate(CharSequence value) throws NumericException { require(value, pos++, n, '-'); int day = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return computeDate(year, month, day, 0, 0, 0, 0, null); + return computeDate(year, month, day, 0, 0, 0, 0, null, 0, 0); } require(value, pos++, n, ' '); int tail = pos; @@ -109,14 +111,15 @@ private static long parsePgDate(CharSequence value) throws NumericException { int millis = 0; if (pos < n && value.charAt(pos) == '.') { long parsed = greedyMillis(value, pos + 1, n); + if (parsed == GREEDY_MILLIS_FAILED) { + throw NumericException.instance(); + } millis = (int) parsed; pos += 1 + (int) (parsed >>> 32); } - return computeDate(year, month, day, hour, minute, second, millis, - ZoneMatcher.parse(value, pos, n)); + return computeDate(year, month, day, hour, minute, second, millis, value, pos, n); } catch (NumericException ignored) { - return computeDate(year, month, day, 0, 0, 0, 0, - ZoneMatcher.parse(value, tail, n)); + return computeDate(year, month, day, 0, 0, 0, 0, value, tail, n); } } @@ -136,112 +139,130 @@ private static long parseUtcDate(CharSequence value) throws NumericException { int second = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, '.'); int millis = fixedInt(value, pos, pos += 3, n); - return computeDate(year, month, day, hour, minute, second, millis, - ZoneMatcher.parse(value, pos, n)); + return computeDate(year, month, day, hour, minute, second, millis, value, pos, n); } private static long parseIso(CharSequence value) throws NumericException { int n = value.length(); - Parsed p = new Parsed(); - int pos = parseFixedYear(value, 0, n, p); + int pos = n > 0 && value.charAt(0) == '-' ? 5 : 4; + int year = pos == 5 ? -fixedInt(value, 1, pos, n) : fixedInt(value, 0, pos, n); if (pos == n) { - return compute(p, null); + return localMicros(year, 1, 1, 0, 0, 0, 0, 0); } require(value, pos++, n, '-'); if (pos + 3 <= n && value.charAt(pos) == 'W') { - p.week = Numbers.parseInt(value, pos + 1, pos + 3); + int week = Numbers.parseInt(value, pos + 1, pos + 3); if (pos + 3 != n) { throw NumericException.instance(); } - return compute(p, null); + return weekMicros(year, week); } - p.month = fixedInt(value, pos, pos += 2, n); + int month = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return compute(p, null); + return localMicros(year, month, 1, 0, 0, 0, 0, 0); } require(value, pos++, n, '-'); - p.day = fixedInt(value, pos, pos += 2, n); + int day = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return compute(p, null); + return localMicros(year, month, day, 0, 0, 0, 0, 0); } require(value, pos++, n, 'T'); - p.hour = fixedInt(value, pos, pos += 2, n); + int hour = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return compute(p, null); + return localMicros(year, month, day, hour, 0, 0, 0, 0); } require(value, pos++, n, ':'); - p.minute = fixedInt(value, pos, pos += 2, n); + int minute = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, ':'); - p.second = fixedInt(value, pos, pos += 2, n); + int second = fixedInt(value, pos, pos += 2, n); if (pos < n && value.charAt(pos) == '.') { - pos++; + return parseIsoFraction(value, pos + 1, n, year, month, day, hour, minute, second); + } + return zoned(localMicros(year, month, day, hour, minute, second, 0, 0), year, value, pos, n); + } + + /** + * Tries the server's fraction layouts in its order: greedy S to SSS, then + * SSSUUU, then S followed by greedy S to SSS, then fixed SSS. A layout whose + * fraction, fields or zone fails hands over to the next one. + */ + private static long parseIsoFraction( + CharSequence value, + int pos, + int n, + int year, + int month, + int day, + int hour, + int minute, + int second + ) throws NumericException { + long parsed = greedyMillis(value, pos, n); + if (parsed != GREEDY_MILLIS_FAILED) { try { - Parsed copy = new Parsed(p); - long parsed = greedyMillis(value, pos, n); - copy.millis = (int) parsed; - return compute(copy, ZoneMatcher.parse(value, pos + (int) (parsed >>> 32), n)); + long local = localMicros(year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(local, year, value, pos + (int) (parsed >>> 32), n); } catch (NumericException ignored) { } - if (pos + 6 <= n) { - try { - Parsed copy = new Parsed(p); - copy.millis = fixedInt(value, pos, pos + 3, n); - copy.micros = fixedInt(value, pos + 3, pos + 6, n); - return compute(copy, ZoneMatcher.parse(value, pos + 6, n)); - } catch (NumericException ignored) { - } + } + if (pos + 6 <= n) { + try { + int millis = fixedInt(value, pos, pos + 3, n); + int micros = fixedInt(value, pos + 3, pos + 6, n); + return zoned(localMicros(year, month, day, hour, minute, second, millis, micros), year, value, pos + 6, n); + } catch (NumericException ignored) { } - if (pos + 2 <= n) { - try { - fixedInt(value, pos, pos + 1, n); - Parsed copy = new Parsed(p); - long parsed = greedyMillis(value, pos + 1, n); - copy.millis = (int) parsed; - return compute(copy, ZoneMatcher.parse(value, pos + 1 + (int) (parsed >>> 32), n)); - } catch (NumericException ignored) { + } + if (pos + 2 <= n) { + try { + fixedInt(value, pos, pos + 1, n); + parsed = greedyMillis(value, pos + 1, n); + if (parsed != GREEDY_MILLIS_FAILED) { + long local = localMicros(year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(local, year, value, pos + 1 + (int) (parsed >>> 32), n); } + } catch (NumericException ignored) { } - // Final UTC_PATTERN fallback is fixed SSS. It intentionally follows - // the greedy S and SS forms, and therefore accepts parseInt forms - // such as +12 and 1_2 as well as a numeric-zone suffix after 3 chars. - if (pos + 3 <= n) { - try { - Parsed copy = new Parsed(p); - copy.millis = fixedInt(value, pos, pos + 3, n); - return compute(copy, ZoneMatcher.parse(value, pos + 3, n)); - } catch (NumericException ignored) { - } + } + // Final UTC_PATTERN fallback is fixed SSS. It intentionally follows + // the greedy S and SS forms, and therefore accepts parseInt forms + // such as +12 and 1_2 as well as a numeric-zone suffix after 3 chars. + if (pos + 3 <= n) { + try { + int millis = fixedInt(value, pos, pos + 3, n); + return zoned(localMicros(year, month, day, hour, minute, second, millis, 0), year, value, pos + 3, n); + } catch (NumericException ignored) { } - throw NumericException.instance(); } - return compute(p, ZoneMatcher.parse(value, pos, n)); + throw NumericException.instance(); } - private static long parsePg(CharSequence value) throws NumericException { + private static long parsePg(CharSequence value, int dash) throws NumericException { int n = value.length(); - int dash = indexOf(value, '-', value.length() > 0 && value.charAt(0) == '-' ? 1 : 0, n); - if (dash < 1) { - throw NumericException.instance(); - } - Parsed p = new Parsed(); int year = greedyInt(value, 0, dash); - p.year = dash == 2 ? REFERENCE_CENTURY + year : year; + if (dash == 2) { + year += REFERENCE_CENTURY; + } int pos = dash + 1; - p.month = fixedInt(value, pos, pos += 2, n); + int month = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, '-'); - p.day = fixedInt(value, pos, pos += 2, n); + int day = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, ' '); - p.hour = fixedInt(value, pos, pos += 2, n); + int hour = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, ':'); - p.minute = fixedInt(value, pos, pos += 2, n); + int minute = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, ':'); - p.second = fixedInt(value, pos, pos += 2, n); + int second = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, '.'); - p.millis = fixedInt(value, pos, pos += 3, n); - return compute(p, ZoneMatcher.parse(value, pos, n)); + int millis = fixedInt(value, pos, pos += 3, n); + return zoned(localMicros(year, month, day, hour, minute, second, millis, 0), year, value, pos, n); } - private static long greedyMillis(CharSequence value, int lo, int hi) throws NumericException { + /** + * Returns {@code (width << 32) | millis} for a greedy 1-3 digit fraction with + * an optional minus sign, or {@link #GREEDY_MILLIS_FAILED}. + */ + private static long greedyMillis(CharSequence value, int lo, int hi) { int p = lo; boolean negative = p < hi && value.charAt(p) == '-'; if (negative) { @@ -254,7 +275,7 @@ private static long greedyMillis(CharSequence value, int lo, int hi) throws Nume } int width = p - lo; if (p == digitStart || width > 3) { - throw NumericException.instance(); + return GREEDY_MILLIS_FAILED; } while (width < 3) { parsed *= 10; @@ -263,27 +284,50 @@ private static long greedyMillis(CharSequence value, int lo, int hi) throws Nume return ((long) (p - lo) << 32) | ((negative ? -parsed : parsed) & 0xffffffffL); } - private static long compute(Parsed p, ZoneMatcher.Zone zone) throws NumericException { - boolean leap = CommonUtils.isLeapYear(p.year); - if (p.month < 1 || p.month > 12 || p.day < 1 || p.day > CommonUtils.getDaysPerMonth(p.month, leap) - || p.hour < 0 || p.hour > 24 || p.minute < 0 || p.minute > 59 - || p.second < 0 || p.second > 59 || (p.week != -1 && (p.week < 1 || p.week > weeks(p.year)))) { + private static long localMicros( + int year, + int month, + int day, + int hour, + int minute, + int second, + int millis, + int micros + ) throws NumericException { + boolean leap = CommonUtils.isLeapYear(year); + if (month < 1 || month > 12 || day < 1 || day > CommonUtils.getDaysPerMonth(month, leap) + || hour < 0 || hour > 24 || minute < 0 || minute > 59 + || second < 0 || second > 59) { throw NumericException.instance(); } - if (p.week != -1) { - long first = yearMicros(p.year, CommonUtils.isLeapYear(p.year)) - + (p.week - 1L) * WEEK + isoYearDayOffset(p.year) * DAY; - int actualYear = year(first); - p.month = monthOfYear(first, actualYear, CommonUtils.isLeapYear(actualYear)); - p.year += p.week == 1 && isoYearDayOffset(p.year) < 0 ? -1 : 0; - p.day = dayOfMonth(first, p.year, p.month, CommonUtils.isLeapYear(p.year)); - // GenericMicrosFormat retains the originally parsed year's leap flag - // after an ISO week crosses a calendar-year boundary. + return yearMicros(year, leap) + monthMicros(month, leap) + (day - 1L) * DAY + + (hour % 24L) * HOUR + minute * MINUTE + second * SECOND + + millis * 1000L + micros; + } + + private static long weekMicros(int year, int week) throws NumericException { + if (week == -1) { + // The server's "no week" sentinel is -1, so an explicit W-1 parses + // as the bare year. + return localMicros(year, 1, 1, 0, 0, 0, 0, 0); } - long out = yearMicros(p.year, leap) + monthMicros(p.month, leap) + (p.day - 1L) * DAY - + (p.hour % 24L) * HOUR + p.minute * MINUTE + p.second * SECOND - + p.millis * 1000L + p.micros; - return zone == null ? out : out - zone.offset(out, p.year); + if (week < 1 || week > weeks(year)) { + throw NumericException.instance(); + } + boolean leap = CommonUtils.isLeapYear(year); + long first = yearMicros(year, leap) + (week - 1L) * WEEK + isoYearDayOffset(year) * DAY; + int actualYear = year(first); + int month = monthOfYear(first, actualYear, CommonUtils.isLeapYear(actualYear)); + int weekYear = year + (week == 1 && isoYearDayOffset(year) < 0 ? -1 : 0); + int day = dayOfMonth(first, weekYear, month, CommonUtils.isLeapYear(weekYear)); + // GenericMicrosFormat retains the originally parsed year's leap flag + // after an ISO week crosses a calendar-year boundary. + return yearMicros(weekYear, leap) + monthMicros(month, leap) + (day - 1L) * DAY; + } + + /** Converts local micros to UTC using the zone text in {@code [lo, hi)}. */ + private static long zoned(long local, int year, CharSequence value, int lo, int hi) throws NumericException { + return local - ZoneMatcher.offset(value, lo, hi, local, year); } private static long computeDate( @@ -294,7 +338,9 @@ private static long computeDate( int minute, int second, int millis, - ZoneMatcher.Zone zone + CharSequence zone, + int zoneLo, + int zoneHi ) throws NumericException { boolean leap = CommonUtils.isLeapYear(year); if (month < 1 || month > 12 || day < 1 || day > CommonUtils.getDaysPerMonth(month, leap) @@ -304,15 +350,7 @@ private static long computeDate( } long out = dateYearMillis(year, leap) + dateMonthMillis(month, leap) + (day - 1L) * DATE_DAY + (hour % 24L) * DATE_HOUR + minute * DATE_MINUTE + second * DATE_SECOND + millis; - return zone == null ? out : out - zone.offsetMillis(out, year); - } - - private static int parseFixedYear(CharSequence s, int pos, int hi, Parsed p) throws NumericException { - int width = pos < hi && s.charAt(pos) == '-' ? 5 : 4; - p.year = width == 5 - ? -fixedInt(s, pos + 1, pos + width, hi) - : fixedInt(s, pos, pos + width, hi); - return pos + width; + return zone == null ? out : out - ZoneMatcher.offsetMillis(zone, zoneLo, zoneHi, out, year); } private static int fixedInt(CharSequence s, int lo, int hi, int limit) throws NumericException { @@ -511,50 +549,33 @@ private static long previousOrSame(long micros, int dow) { return current >= dow ? micros - (current - dow) * DAY : micros - (7 + current - dow) * DAY; } - private static final class Parsed { - private int day = 1; - private int hour; - private int micros; - private int millis; - private int minute; - private int month = 1; - private int second; - private int week = -1; - private int year; - - private Parsed() { - } - - private Parsed(Parsed that) { - day = that.day; - hour = that.hour; - micros = that.micros; - millis = that.millis; - minute = that.minute; - month = that.month; - second = that.second; - week = that.week; - year = that.year; - } - } - private static final class ZoneMatcher { - private static final List TOKENS = tokens(); - private static Zone parse(CharSequence value, int lo, int hi) throws NumericException { + /** + * TIMESTAMP zone offset in micros. The server matches TIMESTAMP zone + * text against UTF-8 bytes, so a non-ASCII name never matches. + */ + private static long offset(CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { if (lo >= hi) { throw NumericException.instance(); } - int minutes = numericOffset(value, lo, hi); + int minutes = numericOffset(s, lo, hi); if (minutes != Integer.MIN_VALUE) { - return new Zone(minutes * MINUTE); + return minutes * MINUTE; } - for (Token token : TOKENS) { - if (token.text.length() == hi - lo && regionMatches(value, lo, token.text)) { - return token.zone; - } + return NamedZones.find(s, lo, hi, true).offset(epoch, year); + } + + /** DATE zone offset in millis, matching zone names case-insensitively as UTF-16. */ + private static long offsetMillis(CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); } - throw NumericException.instance(); + int minutes = numericOffset(s, lo, hi); + if (minutes != Integer.MIN_VALUE) { + return minutes * DATE_MINUTE; + } + return NamedZones.find(s, lo, hi, false).offsetMillis(epoch, year); } private static int numericOffset(CharSequence s, int lo, int hi) { @@ -593,55 +614,10 @@ private static int numericOffset(CharSequence s, int lo, int hi) { : Integer.MIN_VALUE; } - private static List tokens() { - List result = new ArrayList<>(); - Set seen = new java.util.HashSet<>(); - Map rules = new HashMap<>(); - for (String id : ZoneRulesProvider.getAvailableZoneIds()) { - rules.put(id, new Zone(ZoneRulesProvider.getRules(id, true))); - } - for (Map.Entry alias : ZoneId.SHORT_IDS.entrySet()) { - if (!rules.containsKey(alias.getKey())) { - Zone target = rules.get(alias.getValue()); - if (target == null) { - target = new Zone(ZoneId.of(alias.getValue()).getRules()); - } - rules.put(alias.getKey(), target); - } - } - add(result, seen, rules.get("UTC"), "UTC"); - String[][] names = new DateFormatSymbols(Locale.ENGLISH).getZoneStrings(); - for (String[] row : names) { - if (row.length == 0 || !rules.containsKey(row[0])) { - continue; - } - for (String name : row) { - add(result, seen, rules.get(row[0]), name); - } - } - result.sort(Comparator.comparingInt((Token t) -> t.text.length()).reversed()); - return result; - } - - private static void add(List tokens, Set seen, Zone zone, String name) { - if (zone != null && name != null && !name.isEmpty() && seen.add(name)) { - tokens.add(new Token(name.toUpperCase(), zone)); - } - } - private static boolean digit(char c) { return c >= '0' && c <= '9'; } - private static boolean regionMatches(CharSequence value, int lo, String token) { - for (int i = 0; i < token.length(); i++) { - if (Character.toUpperCase(value.charAt(lo + i)) != token.charAt(i)) { - return false; - } - } - return true; - } - private static boolean startsWithIgnoreCase(CharSequence value, int lo, int hi, String prefix) { if (hi - lo < prefix.length()) { return false; @@ -654,13 +630,106 @@ private static boolean startsWithIgnoreCase(CharSequence value, int lo, int hi, return true; } - private static final class Token { - private final String text; - private final Zone zone; + /** + * Zone names by case-insensitive text. A holder class, so that only a + * named-zone lookup loads every zone's rules; 'Z' and numeric offsets + * never initialize it. + */ + private static final class NamedZones { + private static final int MASK; + private static final String[] NAMES; + private static final Zone[] ZONES; + + static { + Map rules = new HashMap<>(); + for (String id : ZoneRulesProvider.getAvailableZoneIds()) { + rules.put(id, new Zone(ZoneRulesProvider.getRules(id, true))); + } + for (Map.Entry alias : ZoneId.SHORT_IDS.entrySet()) { + if (!rules.containsKey(alias.getKey())) { + Zone target = rules.get(alias.getValue()); + if (target == null) { + target = new Zone(ZoneId.of(alias.getValue()).getRules()); + } + rules.put(alias.getKey(), target); + } + } + String[][] rows = new DateFormatSymbols(Locale.ENGLISH).getZoneStrings(); + int names = 1; + for (String[] row : rows) { + names += row.length; + } + // At most half full, so a miss ends at an empty slot quickly. + int capacity = 16; + while (capacity < 2 * names) { + capacity <<= 1; + } + MASK = capacity - 1; + NAMES = new String[capacity]; + ZONES = new Zone[capacity]; + // The first definition of a name wins; UTC goes first. + define(rules.get("UTC"), "UTC"); + for (String[] row : rows) { + if (row.length == 0 || !rules.containsKey(row[0])) { + continue; + } + Zone zone = rules.get(row[0]); + for (String name : row) { + define(zone, name); + } + } + } + + private static void define(Zone zone, String name) { + if (zone == null || name == null || name.isEmpty()) { + return; + } + String text = name.toUpperCase(Locale.ROOT); + int hash = 0; + for (int i = 0, n = text.length(); i < n; i++) { + hash = 31 * hash + text.charAt(i); + } + int slot = spread(hash) & MASK; + while (NAMES[slot] != null) { + if (NAMES[slot].equals(text)) { + return; + } + slot = (slot + 1) & MASK; + } + NAMES[slot] = text; + ZONES[slot] = zone; + } + + private static Zone find(CharSequence s, int lo, int hi, boolean isAsciiOnly) throws NumericException { + int hash = 0; + for (int i = lo; i < hi; i++) { + char c = s.charAt(i); + if (isAsciiOnly && c > 0x7f) { + throw NumericException.instance(); + } + hash = 31 * hash + Character.toUpperCase(c); + } + int length = hi - lo; + for (int slot = spread(hash) & MASK; NAMES[slot] != null; slot = (slot + 1) & MASK) { + String name = NAMES[slot]; + if (name.length() == length && regionMatches(s, lo, name)) { + return ZONES[slot]; + } + } + throw NumericException.instance(); + } + + private static boolean regionMatches(CharSequence value, int lo, String name) { + for (int i = 0, n = name.length(); i < n; i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != name.charAt(i)) { + return false; + } + } + return true; + } - private Token(String text, Zone zone) { - this.text = text; - this.zone = zone; + private static int spread(int hash) { + return hash ^ (hash >>> 16); } } @@ -697,7 +766,8 @@ private long offset(long epoch, int year) { // The server computes future rules with the parsed year, // even when calendar arithmetic has wrapped the epoch. long after = 0; - for (ZoneOffsetTransitionRule rule : recurring) { + for (int i = 0, n = recurring.size(); i < n; i++) { + ZoneOffsetTransitionRule rule = recurring.get(i); long transition = transitionEpoch(rule, year); long before = rule.getOffsetBefore().getTotalSeconds() * SECOND; if (epoch < transition) { @@ -728,7 +798,8 @@ private long offsetMillis(long epoch, int year) throws NumericException { throw NumericException.instance(); } long after = 0; - for (ZoneOffsetTransitionRule rule : recurring) { + for (int i = 0, n = recurring.size(); i < n; i++) { + ZoneOffsetTransitionRule rule = recurring.get(i); long transition = transitionEpochMillis(rule, year); long before = rule.getOffsetBefore().getTotalSeconds() * DATE_SECOND; if (epoch < transition) { From fb28ffe583df18a02bb0a64284e8c8e856a23528 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 13:50:27 +0200 Subject: [PATCH 39/51] Merge micros and millis timestamp parser paths QwpSchemaTimestampParser carried a micros copy (TIMESTAMP) and a millis copy (DATE) of most of its arithmetic: local time, year and month starts, day of week, next/previous weekday, zone offsets and recurring zone transitions. The copies differed only in their unit and in three behaviours of the server's two parser families. A Target now holds the unit sizes plus those three differences as named flags: micros year starts saturate on negative overflow, TIMESTAMP zone names must be ASCII (the server matches them as UTF-8 bytes), and DATE recurring zone rules reject negative years. Each function pair becomes one function over a Target. Zone stores its cutoff and offsets in seconds and scales them per unit; the scaling reproduces the old per-unit values exactly, including Instant.ofEpochMilli for historic DATE offsets. Month tables hold day counts instead of per-unit values. The PG DATE tail fallback and the UTC DATE layout stay DATE-specific. The file shrinks from 889 to 752 lines. Flipping any one Target flag fails a pinned corpus case (micro_parsed_long_min, min_int_year, negative_year_named_zone_wrap, micro_nonascii_zone_lookalike). The client corpora, the OSS e2e suites against a real server (22 tests) and 2471 QWP client tests pass; the binding tests also pass on JDK 8. --- .../protocol/QwpSchemaTimestampParser.java | 427 ++++++------------ 1 file changed, 145 insertions(+), 282 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java index 03be1d108..ed1a57b3b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java @@ -29,22 +29,16 @@ * this is deliberately not a general date-format compiler. */ final class QwpSchemaTimestampParser { - private static final long DAY = 86_400_000_000L; - private static final long GREEDY_MILLIS_FAILED = -1L; - private static final long DATE_DAY = 86_400_000L; - private static final long DATE_HOUR = 3_600_000L; - private static final long DATE_MINUTE = 60_000L; - private static final long DATE_SECOND = 1_000L; - private static final long[] DATE_MONTH_COMMON = dateMonthStarts(false); - private static final long[] DATE_MONTH_LEAP = dateMonthStarts(true); private static final int DAYS_0000_TO_1970 = 719527; - private static final long HOUR = 3_600_000_000L; - private static final long MINUTE = 60_000_000L; - private static final long SECOND = 1_000_000L; - private static final long WEEK = 7 * DAY; - private static final long[] MONTH_COMMON = monthStarts(false); - private static final long[] MONTH_LEAP = monthStarts(true); + private static final long GREEDY_MILLIS_FAILED = -1L; + private static final int[] MONTH_START_DAYS_COMMON = monthStartDays(false); + private static final int[] MONTH_START_DAYS_LEAP = monthStartDays(true); private static final int REFERENCE_CENTURY = referenceCentury(); + // The server parses TIMESTAMP text with MicrosFormatUtils over UTF-8 bytes, + // and DATE text with DateFormatUtils over UTF-16. Its two parsers also differ + // in year overflow and in zone rules for negative years. + private static final Target DATE = new Target(1L, false, false, true); + private static final Target TIMESTAMP = new Target(1_000L, true, true, false); private QwpSchemaTimestampParser() { } @@ -98,7 +92,7 @@ private static long parsePgDate(CharSequence value, int dash) throws NumericExce require(value, pos++, n, '-'); int day = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return computeDate(year, month, day, 0, 0, 0, 0, null, 0, 0); + return localTime(DATE, year, month, day, 0, 0, 0, 0, 0); } require(value, pos++, n, ' '); int tail = pos; @@ -117,9 +111,9 @@ private static long parsePgDate(CharSequence value, int dash) throws NumericExce millis = (int) parsed; pos += 1 + (int) (parsed >>> 32); } - return computeDate(year, month, day, hour, minute, second, millis, value, pos, n); + return zoned(DATE, localTime(DATE, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); } catch (NumericException ignored) { - return computeDate(year, month, day, 0, 0, 0, 0, value, tail, n); + return zoned(DATE, localTime(DATE, year, month, day, 0, 0, 0, 0, 0), year, value, tail, n); } } @@ -139,7 +133,7 @@ private static long parseUtcDate(CharSequence value) throws NumericException { int second = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, '.'); int millis = fixedInt(value, pos, pos += 3, n); - return computeDate(year, month, day, hour, minute, second, millis, value, pos, n); + return zoned(DATE, localTime(DATE, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); } private static long parseIso(CharSequence value) throws NumericException { @@ -147,7 +141,7 @@ private static long parseIso(CharSequence value) throws NumericException { int pos = n > 0 && value.charAt(0) == '-' ? 5 : 4; int year = pos == 5 ? -fixedInt(value, 1, pos, n) : fixedInt(value, 0, pos, n); if (pos == n) { - return localMicros(year, 1, 1, 0, 0, 0, 0, 0); + return localTime(TIMESTAMP, year, 1, 1, 0, 0, 0, 0, 0); } require(value, pos++, n, '-'); if (pos + 3 <= n && value.charAt(pos) == 'W') { @@ -159,17 +153,17 @@ private static long parseIso(CharSequence value) throws NumericException { } int month = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return localMicros(year, month, 1, 0, 0, 0, 0, 0); + return localTime(TIMESTAMP, year, month, 1, 0, 0, 0, 0, 0); } require(value, pos++, n, '-'); int day = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return localMicros(year, month, day, 0, 0, 0, 0, 0); + return localTime(TIMESTAMP, year, month, day, 0, 0, 0, 0, 0); } require(value, pos++, n, 'T'); int hour = fixedInt(value, pos, pos += 2, n); if (pos == n) { - return localMicros(year, month, day, hour, 0, 0, 0, 0); + return localTime(TIMESTAMP, year, month, day, hour, 0, 0, 0, 0); } require(value, pos++, n, ':'); int minute = fixedInt(value, pos, pos += 2, n); @@ -178,7 +172,7 @@ private static long parseIso(CharSequence value) throws NumericException { if (pos < n && value.charAt(pos) == '.') { return parseIsoFraction(value, pos + 1, n, year, month, day, hour, minute, second); } - return zoned(localMicros(year, month, day, hour, minute, second, 0, 0), year, value, pos, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, 0, 0), year, value, pos, n); } /** @@ -200,8 +194,8 @@ private static long parseIsoFraction( long parsed = greedyMillis(value, pos, n); if (parsed != GREEDY_MILLIS_FAILED) { try { - long local = localMicros(year, month, day, hour, minute, second, (int) parsed, 0); - return zoned(local, year, value, pos + (int) (parsed >>> 32), n); + long local = localTime(TIMESTAMP, year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(TIMESTAMP, local, year, value, pos + (int) (parsed >>> 32), n); } catch (NumericException ignored) { } } @@ -209,7 +203,7 @@ private static long parseIsoFraction( try { int millis = fixedInt(value, pos, pos + 3, n); int micros = fixedInt(value, pos + 3, pos + 6, n); - return zoned(localMicros(year, month, day, hour, minute, second, millis, micros), year, value, pos + 6, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, micros), year, value, pos + 6, n); } catch (NumericException ignored) { } } @@ -218,8 +212,8 @@ private static long parseIsoFraction( fixedInt(value, pos, pos + 1, n); parsed = greedyMillis(value, pos + 1, n); if (parsed != GREEDY_MILLIS_FAILED) { - long local = localMicros(year, month, day, hour, minute, second, (int) parsed, 0); - return zoned(local, year, value, pos + 1 + (int) (parsed >>> 32), n); + long local = localTime(TIMESTAMP, year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(TIMESTAMP, local, year, value, pos + 1 + (int) (parsed >>> 32), n); } } catch (NumericException ignored) { } @@ -230,7 +224,7 @@ private static long parseIsoFraction( if (pos + 3 <= n) { try { int millis = fixedInt(value, pos, pos + 3, n); - return zoned(localMicros(year, month, day, hour, minute, second, millis, 0), year, value, pos + 3, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, 0), year, value, pos + 3, n); } catch (NumericException ignored) { } } @@ -255,7 +249,7 @@ private static long parsePg(CharSequence value, int dash) throws NumericExceptio int second = fixedInt(value, pos, pos += 2, n); require(value, pos++, n, '.'); int millis = fixedInt(value, pos, pos += 3, n); - return zoned(localMicros(year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); } /** @@ -284,7 +278,9 @@ private static long greedyMillis(CharSequence value, int lo, int hi) { return ((long) (p - lo) << 32) | ((negative ? -parsed : parsed) & 0xffffffffL); } - private static long localMicros( + /** Local time in {@code t}'s unit, from validated fields. */ + private static long localTime( + Target t, int year, int month, int day, @@ -300,57 +296,35 @@ private static long localMicros( || second < 0 || second > 59) { throw NumericException.instance(); } - return yearMicros(year, leap) + monthMicros(month, leap) + (day - 1L) * DAY - + (hour % 24L) * HOUR + minute * MINUTE + second * SECOND - + millis * 1000L + micros; + return yearStart(t, year, leap) + monthStart(t, month, leap) + (day - 1L) * t.day + + (hour % 24L) * t.hour + minute * t.minute + second * t.second + + millis * t.milli + micros; } private static long weekMicros(int year, int week) throws NumericException { if (week == -1) { // The server's "no week" sentinel is -1, so an explicit W-1 parses // as the bare year. - return localMicros(year, 1, 1, 0, 0, 0, 0, 0); + return localTime(TIMESTAMP, year, 1, 1, 0, 0, 0, 0, 0); } if (week < 1 || week > weeks(year)) { throw NumericException.instance(); } boolean leap = CommonUtils.isLeapYear(year); - long first = yearMicros(year, leap) + (week - 1L) * WEEK + isoYearDayOffset(year) * DAY; + long first = yearStart(TIMESTAMP, year, leap) + (week - 1L) * 7 * TIMESTAMP.day + + isoYearDayOffset(year) * TIMESTAMP.day; int actualYear = year(first); int month = monthOfYear(first, actualYear, CommonUtils.isLeapYear(actualYear)); int weekYear = year + (week == 1 && isoYearDayOffset(year) < 0 ? -1 : 0); int day = dayOfMonth(first, weekYear, month, CommonUtils.isLeapYear(weekYear)); // GenericMicrosFormat retains the originally parsed year's leap flag // after an ISO week crosses a calendar-year boundary. - return yearMicros(weekYear, leap) + monthMicros(month, leap) + (day - 1L) * DAY; + return yearStart(TIMESTAMP, weekYear, leap) + monthStart(TIMESTAMP, month, leap) + (day - 1L) * TIMESTAMP.day; } - /** Converts local micros to UTC using the zone text in {@code [lo, hi)}. */ - private static long zoned(long local, int year, CharSequence value, int lo, int hi) throws NumericException { - return local - ZoneMatcher.offset(value, lo, hi, local, year); - } - - private static long computeDate( - int year, - int month, - int day, - int hour, - int minute, - int second, - int millis, - CharSequence zone, - int zoneLo, - int zoneHi - ) throws NumericException { - boolean leap = CommonUtils.isLeapYear(year); - if (month < 1 || month > 12 || day < 1 || day > CommonUtils.getDaysPerMonth(month, leap) - || hour < 0 || hour > 24 || minute < 0 || minute > 59 - || second < 0 || second > 59) { - throw NumericException.instance(); - } - long out = dateYearMillis(year, leap) + dateMonthMillis(month, leap) + (day - 1L) * DATE_DAY - + (hour % 24L) * DATE_HOUR + minute * DATE_MINUTE + second * DATE_SECOND + millis; - return zone == null ? out : out - ZoneMatcher.offsetMillis(zone, zoneLo, zoneHi, out, year); + /** Converts {@code local}, in {@code t}'s unit, to UTC using the zone text in {@code [lo, hi)}. */ + private static long zoned(Target t, long local, int year, CharSequence value, int lo, int hi) throws NumericException { + return local - ZoneMatcher.offset(t, value, lo, hi, local, year); } private static int fixedInt(CharSequence s, int lo, int hi, int limit) throws NumericException { @@ -407,71 +381,19 @@ private static int referenceCentury() { return offset + 20 > 100 ? year - offset + 100 : year - offset; } - private static long[] monthStarts(boolean leap) { - long[] result = new long[12]; - for (int i = 1; i < 12; i++) { - result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap) * DAY; - } - return result; - } - - private static long[] dateMonthStarts(boolean leap) { - long[] result = new long[12]; + private static int[] monthStartDays(boolean leap) { + int[] result = new int[12]; for (int i = 1; i < 12; i++) { - result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap) * DATE_DAY; + result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap); } return result; } - - private static long dateMonthMillis(int month, boolean leap) { - return (leap ? DATE_MONTH_LEAP : DATE_MONTH_COMMON)[month - 1]; + private static long monthStart(Target t, int month, boolean leap) { + return (leap ? MONTH_START_DAYS_LEAP : MONTH_START_DAYS_COMMON)[month - 1] * t.day; } - - private static long dateYearMillis(int year, boolean leap) { - int leapYears = year / 100; - if (year < 0) { - leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; - } else { - leapYears = (year >> 2) - leapYears + (leapYears >> 2); - if (leap) { - leapYears--; - } - } - return (year * 365L + leapYears - DAYS_0000_TO_1970) * DATE_DAY; - } - - private static int dateDayOfWeek(long millis) { - long d; - if (millis > -1) { - d = millis / DATE_DAY; - } else { - d = (millis - (DATE_DAY - 1)) / DATE_DAY; - if (d < -3) { - return 7 + (int) ((d + 4) % 7); - } - } - return 1 + (int) ((d + 3) % 7); - } - - private static long nextDateOrSame(long millis, int dow) { - int current = dateDayOfWeek(millis); - return current <= dow ? millis + (dow - current) * DATE_DAY - : millis + (7 - current + dow) * DATE_DAY; - } - - private static long previousDateOrSame(long millis, int dow) { - int current = dateDayOfWeek(millis); - return current >= dow ? millis - (current - dow) * DATE_DAY - : millis - (7 + current - dow) * DATE_DAY; - } - - private static long monthMicros(int month, boolean leap) { - return (leap ? MONTH_LEAP : MONTH_COMMON)[month - 1]; - } - - private static long yearMicros(int year, boolean leap) { - // Preserve server Micros.yearMicros arithmetic, including unchecked - // multiplication and its asymmetric negative-overflow saturation. + private static long yearStart(Target t, int year, boolean leap) { + // Preserve the server's unchecked multiplication. Only its micros + // variant saturates on negative overflow, and asymmetrically. int leapYears = year / 100; if (year < 0) { leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; @@ -482,23 +404,21 @@ private static long yearMicros(int year, boolean leap) { } } long days = year * 365L + leapYears - DAYS_0000_TO_1970; - long micros = days * DAY; - return days < 0 && micros > 0 ? Long.MIN_VALUE : micros; + long start = days * t.day; + return t.isYearSaturating && days < 0 && start > 0 ? Long.MIN_VALUE : start; } - - private static int dayOfWeek(long micros) { + private static int dayOfWeek(Target t, long time) { long d; - if (micros > -1) { - d = micros / DAY; + if (time > -1) { + d = time / t.day; } else { - d = (micros - (DAY - 1)) / DAY; + d = (time - (t.day - 1)) / t.day; if (d < -3) { return 7 + (int) ((d + 4) % 7); } } return 1 + (int) ((d + 3) % 7); } - private static int endYearDow(int year) { return (year + Math.abs(year / 4) - Math.abs(year / 100) + Math.abs(year / 400)) % 7; } @@ -513,69 +433,52 @@ private static int isoYearDayOffset(int year) { } private static int monthOfYear(long micros, int year, boolean leap) { - long elapsed = micros - yearMicros(year, leap); - long[] starts = leap ? MONTH_LEAP : MONTH_COMMON; + long elapsed = micros - yearStart(TIMESTAMP, year, leap); int month = 12; - while (month > 1 && elapsed < starts[month - 1]) { + while (month > 1 && elapsed < monthStart(TIMESTAMP, month, leap)) { month--; } return month; } - private static int year(long micros) { int estimate = 1970 + (int) (micros / 31_556_952_000_000L); if (micros < 0 && estimate >= 1970) { estimate = 1969; } boolean leap = CommonUtils.isLeapYear(estimate); - long diff = micros - yearMicros(estimate, leap); + long diff = micros - yearStart(TIMESTAMP, estimate, leap); if (diff < 0) { return estimate - 1; } - return diff >= (leap ? 366L : 365L) * DAY ? estimate + 1 : estimate; + return diff >= (leap ? 366L : 365L) * TIMESTAMP.day ? estimate + 1 : estimate; } - private static int dayOfMonth(long micros, int year, int month, boolean leap) { - return (int) ((micros - yearMicros(year, leap) - monthMicros(month, leap)) / DAY) + 1; + return (int) ((micros - yearStart(TIMESTAMP, year, leap) - monthStart(TIMESTAMP, month, leap)) / TIMESTAMP.day) + 1; } - - private static long nextOrSame(long micros, int dow) { - int current = dayOfWeek(micros); - return current <= dow ? micros + (dow - current) * DAY : micros + (7 - current + dow) * DAY; + private static long nextOrSame(Target t, long time, int dow) { + int current = dayOfWeek(t, time); + return current <= dow ? time + (dow - current) * t.day : time + (7 - current + dow) * t.day; } - - private static long previousOrSame(long micros, int dow) { - int current = dayOfWeek(micros); - return current >= dow ? micros - (current - dow) * DAY : micros - (7 + current - dow) * DAY; + private static long previousOrSame(Target t, long time, int dow) { + int current = dayOfWeek(t, time); + return current >= dow ? time - (current - dow) * t.day : time - (7 + current - dow) * t.day; } - private static final class ZoneMatcher { /** - * TIMESTAMP zone offset in micros. The server matches TIMESTAMP zone - * text against UTF-8 bytes, so a non-ASCII name never matches. + * Zone offset in {@code t}'s unit. The server matches TIMESTAMP zone text + * against UTF-8 bytes, where a non-ASCII name never matches, and DATE zone + * text case-insensitively as UTF-16. */ - private static long offset(CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { - if (lo >= hi) { - throw NumericException.instance(); - } - int minutes = numericOffset(s, lo, hi); - if (minutes != Integer.MIN_VALUE) { - return minutes * MINUTE; - } - return NamedZones.find(s, lo, hi, true).offset(epoch, year); - } - - /** DATE zone offset in millis, matching zone names case-insensitively as UTF-16. */ - private static long offsetMillis(CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { + private static long offset(Target t, CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { if (lo >= hi) { throw NumericException.instance(); } int minutes = numericOffset(s, lo, hi); if (minutes != Integer.MIN_VALUE) { - return minutes * DATE_MINUTE; + return minutes * t.minute; } - return NamedZones.find(s, lo, hi, false).offsetMillis(epoch, year); + return NamedZones.find(s, lo, hi, t.isZoneAsciiOnly).offset(t, epoch, year); } private static int numericOffset(CharSequence s, int lo, int hi) { @@ -734,156 +637,116 @@ private static int spread(int hash) { } private static final class Zone { - private final long cutoff; - private final long fixedOffset; - private final long lastWall; + // Seconds, or Long.MIN_VALUE when absent. + private final long cutoffSeconds; + private final long fixedOffsetSeconds; + private final long lastWallSeconds; private final List recurring; private final ZoneRules rules; - private Zone(long fixedOffset) { - this.cutoff = Long.MIN_VALUE; - this.fixedOffset = fixedOffset; - this.lastWall = fixedOffset; - this.recurring = java.util.Collections.emptyList(); - this.rules = null; - } - private Zone(ZoneRules rules) { this.rules = rules; List history = rules.getTransitions(); - cutoff = history.isEmpty() ? Long.MIN_VALUE : history.get(history.size() - 1).toEpochSecond() * SECOND; + cutoffSeconds = history.isEmpty() ? Long.MIN_VALUE : history.get(history.size() - 1).toEpochSecond(); recurring = rules.getTransitionRules(); - fixedOffset = rules.isFixedOffset() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() * SECOND : Long.MIN_VALUE; - lastWall = history.isEmpty() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() * SECOND - : history.get(history.size() - 1).getOffsetAfter().getTotalSeconds() * SECOND; + fixedOffsetSeconds = rules.isFixedOffset() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() : Long.MIN_VALUE; + lastWallSeconds = history.isEmpty() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() + : history.get(history.size() - 1).getOffsetAfter().getTotalSeconds(); } - private long offset(long epoch, int year) { - if (fixedOffset != Long.MIN_VALUE) { - return fixedOffset; - } - if (!recurring.isEmpty() && epoch > cutoff) { - // The server computes future rules with the parsed year, - // even when calendar arithmetic has wrapped the epoch. - long after = 0; - for (int i = 0, n = recurring.size(); i < n; i++) { - ZoneOffsetTransitionRule rule = recurring.get(i); - long transition = transitionEpoch(rule, year); - long before = rule.getOffsetBefore().getTotalSeconds() * SECOND; - if (epoch < transition) { - return before; - } - after = rule.getOffsetAfter().getTotalSeconds() * SECOND; - } - return after; - } - if (epoch > cutoff) { - return lastWall; - } - Instant instant = Instant.ofEpochSecond(Math.floorDiv(epoch, SECOND), Math.floorMod(epoch, SECOND) * 1000); - return rules.getOffset(instant).getTotalSeconds() * SECOND; - } - - private long offsetMillis(long epoch, int year) throws NumericException { - if (fixedOffset != Long.MIN_VALUE) { - return fixedOffset / 1000L; - } - long cutoffMillis = cutoff == Long.MIN_VALUE ? Long.MIN_VALUE : cutoff / 1000L; - if (!recurring.isEmpty() && epoch > cutoffMillis) { - // The server's recurring-rule cache rejects negative year keys. This can - // only be reached after full-width DATE arithmetic wraps past its cutoff; - // reject it as an invalid value instead of silently accepting bytes that - // the server's STRING conversion cannot produce. - if (year < 0) { - throw NumericException.instance(); - } - long after = 0; - for (int i = 0, n = recurring.size(); i < n; i++) { - ZoneOffsetTransitionRule rule = recurring.get(i); - long transition = transitionEpochMillis(rule, year); - long before = rule.getOffsetBefore().getTotalSeconds() * DATE_SECOND; - if (epoch < transition) { - return before; - } - after = rule.getOffsetAfter().getTotalSeconds() * DATE_SECOND; - } - return after; - } - if (epoch > cutoffMillis) { - return lastWall / 1000L; - } - return rules.getOffset(Instant.ofEpochMilli(epoch)).getTotalSeconds() * DATE_SECOND; - } - - private static long transitionEpoch(ZoneOffsetTransitionRule rule, int year) { + private static long transitionEpoch(Target t, ZoneOffsetTransitionRule rule, int year) { boolean leap = CommonUtils.isLeapYear(year); int month = rule.getMonth().getValue(); int dom = rule.getDayOfMonthIndicator(); int dow = rule.getDayOfWeek() == null ? -1 : rule.getDayOfWeek().getValue(); + long time = rule.getLocalTime().getHour() * t.hour + rule.getLocalTime().getMinute() * t.minute + + rule.getLocalTime().getSecond() * t.second; long timestamp; if (dom < 0) { - timestamp = yearMicros(year, leap) + monthMicros(month, leap) - + (CommonUtils.getDaysPerMonth(month, leap) + dom) * DAY - + rule.getLocalTime().getHour() * HOUR + rule.getLocalTime().getMinute() * MINUTE - + rule.getLocalTime().getSecond() * SECOND; + timestamp = yearStart(t, year, leap) + monthStart(t, month, leap) + + (CommonUtils.getDaysPerMonth(month, leap) + dom) * t.day + time; if (dow > -1) { - timestamp = previousOrSame(timestamp, dow); + timestamp = previousOrSame(t, timestamp, dow); } } else { - timestamp = yearMicros(year, leap) + monthMicros(month, leap) + (dom - 1L) * DAY - + rule.getLocalTime().getHour() * HOUR + rule.getLocalTime().getMinute() * MINUTE - + rule.getLocalTime().getSecond() * SECOND; + timestamp = yearStart(t, year, leap) + monthStart(t, month, leap) + (dom - 1L) * t.day + time; if (dow > -1) { - timestamp = nextOrSame(timestamp, dow); + timestamp = nextOrSame(t, timestamp, dow); } } if (rule.isMidnightEndOfDay()) { - timestamp += DAY; + timestamp += t.day; } int before = rule.getOffsetBefore().getTotalSeconds(); if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.UTC) { - timestamp += before * SECOND; + timestamp += before * t.second; } else if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.STANDARD) { - timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * SECOND; + timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * t.second; } - return timestamp - before * SECOND; + return timestamp - before * t.second; } - private static long transitionEpochMillis(ZoneOffsetTransitionRule rule, int year) { - boolean leap = CommonUtils.isLeapYear(year); - int month = rule.getMonth().getValue(); - int dom = rule.getDayOfMonthIndicator(); - int dow = rule.getDayOfWeek() == null ? -1 : rule.getDayOfWeek().getValue(); - long timestamp; - if (dom < 0) { - timestamp = dateYearMillis(year, leap) + dateMonthMillis(month, leap) - + (CommonUtils.getDaysPerMonth(month, leap) + dom) * DATE_DAY - + rule.getLocalTime().getHour() * DATE_HOUR - + rule.getLocalTime().getMinute() * DATE_MINUTE - + rule.getLocalTime().getSecond() * DATE_SECOND; - if (dow > -1) { - timestamp = previousDateOrSame(timestamp, dow); + private long offset(Target t, long epoch, int year) throws NumericException { + if (fixedOffsetSeconds != Long.MIN_VALUE) { + return fixedOffsetSeconds * t.second; + } + long cutoff = cutoffSeconds == Long.MIN_VALUE ? Long.MIN_VALUE : cutoffSeconds * t.second; + if (!recurring.isEmpty() && epoch > cutoff) { + if (t.isNegativeYearRuleRejected && year < 0) { + // The server's DATE recurring-rule cache rejects negative + // year keys. This is only reachable after full-width DATE + // arithmetic wraps past the cutoff. + throw NumericException.instance(); } - } else { - timestamp = dateYearMillis(year, leap) + dateMonthMillis(month, leap) + (dom - 1L) * DATE_DAY - + rule.getLocalTime().getHour() * DATE_HOUR - + rule.getLocalTime().getMinute() * DATE_MINUTE - + rule.getLocalTime().getSecond() * DATE_SECOND; - if (dow > -1) { - timestamp = nextDateOrSame(timestamp, dow); + // The server computes future rules with the parsed year, + // even when calendar arithmetic has wrapped the epoch. + long after = 0; + for (int i = 0, n = recurring.size(); i < n; i++) { + ZoneOffsetTransitionRule rule = recurring.get(i); + if (epoch < transitionEpoch(t, rule, year)) { + return rule.getOffsetBefore().getTotalSeconds() * t.second; + } + after = rule.getOffsetAfter().getTotalSeconds() * t.second; } + return after; } - if (rule.isMidnightEndOfDay()) { - timestamp += DATE_DAY; - } - int before = rule.getOffsetBefore().getTotalSeconds(); - if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.UTC) { - timestamp += before * DATE_SECOND; - } else if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.STANDARD) { - timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * DATE_SECOND; + if (epoch > cutoff) { + return lastWallSeconds * t.second; } - return timestamp - before * DATE_SECOND; - } + Instant instant = Instant.ofEpochSecond(Math.floorDiv(epoch, t.second), Math.floorMod(epoch, t.second) * t.nanos); + return rules.getOffset(instant).getTotalSeconds() * t.second; + } + } + } + + /** The server's TIMESTAMP (micros) or DATE (millis) text parser family. */ + private static final class Target { + private final long day; + private final long hour; + private final boolean isNegativeYearRuleRejected; + private final boolean isYearSaturating; + private final boolean isZoneAsciiOnly; + private final long milli; + private final long minute; + // Nanos per unit. + private final long nanos; + private final long second; + + private Target( + long unitsPerMilli, + boolean isYearSaturating, + boolean isZoneAsciiOnly, + boolean isNegativeYearRuleRejected + ) { + this.milli = unitsPerMilli; + this.second = 1_000L * unitsPerMilli; + this.minute = 60_000L * unitsPerMilli; + this.hour = 3_600_000L * unitsPerMilli; + this.day = 86_400_000L * unitsPerMilli; + this.nanos = 1_000_000L / unitsPerMilli; + this.isYearSaturating = isYearSaturating; + this.isZoneAsciiOnly = isZoneAsciiOnly; + this.isNegativeYearRuleRejected = isNegativeYearRuleRejected; } } } From 5c5fca8e879e977a5ac87e4324cf55b74cc4761f Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 14:15:24 +0200 Subject: [PATCH 40/51] Stop schema lookups from waking the I/O thread QwpSchemaCoordinator.resolve() unparked the I/O thread after queueing a DESCRIBE. While the wire was down, that unpark cut connectLoop's backoff park short, so a producer retrying a STRICT lookup drove one reconnect attempt per lookup instead of one per backoff interval: 940 lookups in one second produced 940 connect attempts. The unpark bought nothing. The I/O loop polls hasPendingRequest on every pass, parking at most DEFAULT_PARK_NANOS (50 us) between passes, and connectLoop never services the request anyway. Removing it restores the base invariant that only close() unparks the I/O thread. A lookup now waits up to one idle park before it goes out, which is negligible next to its network round trip. testLookupsDuringOutageDoNotShortenReconnectBackoff drops the wire with a 200 ms backoff and resolves in a tight loop for one second; attempts stay within the backoff budget. 2472 QWP client tests pass. --- .../sf/cursor/CursorWebSocketSendLoop.java | 4 +- .../sf/cursor/QwpSchemaCoordinator.java | 6 +-- .../CursorWebSocketSchemaLifecycleTest.java | 50 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 5db8cf52d..147870bd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1227,7 +1227,7 @@ public void checkError() { * store-and-forward state, or acknowledgment watermarks. */ public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMillis) { - return schemaCoordinator.resolve(tableName, timeoutMillis, false, ioThread); + return schemaCoordinator.resolve(tableName, timeoutMillis, false); } /** @@ -1256,7 +1256,7 @@ public void setAbandonSchemaLookupOnDisconnect(boolean abandon) { * This does not mutate rows, store-and-forward state, or acknowledgment watermarks. */ public QwpSchemaResponse refreshSchema(CharSequence tableName, long timeoutMillis) { - return schemaCoordinator.resolve(tableName, timeoutMillis, true, ioThread); + return schemaCoordinator.resolve(tableName, timeoutMillis, true); } /** diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index 951bb95b8..9f6433c5f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -15,7 +15,6 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; -import java.util.concurrent.locks.LockSupport; import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; @@ -29,7 +28,7 @@ final class QwpSchemaCoordinator { private long nextRequestId = 1; private Request request; - QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean refresh, Thread ioThread) { + QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean refresh) { final long startNanos = System.nanoTime(); if (timeoutMillis < 0) { throw new IllegalArgumentException("timeoutMillis must be non-negative"); @@ -67,8 +66,9 @@ QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean re nextRequestId++; Request own = new Request(id, key, tableName.toString(), startNanos, timeoutNanos, message); request = own; + // The I/O loop polls hasPendingRequest on every pass; waking it would + // cut its reconnect backoff short while the wire is down. hasPendingRequest = true; - LockSupport.unpark(ioThread); while (!own.done) { try { await(own, key); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java index 9d928f9cb..19d1f6484 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -36,6 +36,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; public class CursorWebSocketSchemaLifecycleTest { @@ -307,6 +308,43 @@ public void testCloseDuringReconnectReleasesLookupBeforeFactoryReturns() throws } } + @Test + public void testLookupsDuringOutageDoNotShortenReconnectBackoff() throws Exception { + SchemaHandler handler = new SchemaHandler(); + FailingReconnectFactory reconnect = new FailingReconnectFactory(); + try (CursorSendEngine engine = engine("outage-backoff")) { + TestWebSocketServer server = server(handler); + CursorWebSocketSendLoop loop = null; + try { + loop = new CursorWebSocketSendLoop(connect(server.getPort()), engine, 0, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, reconnect, 200, 200); + loop.start(); + server.close(); + Assert.assertTrue(reconnect.firstAttempt.await(5, TimeUnit.SECONDS)); + int attemptsBefore = reconnect.attempts.get(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + int lookups = 0; + while (System.nanoTime() < deadline) { + try { + loop.resolveSchema("outage", 1); + Assert.fail("the wire is down"); + } catch (LineSenderSchemaException expected) { + lookups++; + } + } + int attempts = reconnect.attempts.get() - attemptsBefore; + // A 200 ms backoff plus up to 200 ms jitter allows at most 5 attempts per second. + Assert.assertTrue("lookups=" + lookups + ", attempts=" + attempts, attempts <= 6); + Assert.assertTrue("lookups=" + lookups, lookups > 50); + } finally { + if (loop != null) { + loop.close(); + } + server.close(); + } + } + } + @Test public void testMalformedCurrentReplyCancelsLookupEvenWithReconnectAvailable() throws Exception { SchemaHandler handler = new SchemaHandler(); @@ -545,6 +583,18 @@ private void join() throws InterruptedException { } } + private static final class FailingReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final CountDownLatch firstAttempt = new CountDownLatch(1); + + @Override + public WebSocketClient reconnect() { + attempts.incrementAndGet(); + firstAttempt.countDown(); + throw new RuntimeException("server is down"); + } + } + private static final class GatedReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { private final CountDownLatch allow = new CountDownLatch(1); private final CountDownLatch attempted = new CountDownLatch(1); From 637fb98737fbf7c35016935f353cd95ed709456e Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 14:26:11 +0200 Subject: [PATCH 41/51] Treat Long.MAX_VALUE wait budgets as unlimited schema_wait_millis, drain(timeoutMillis) and close_flush_timeout_millis each built an absolute deadline as nanoTime() + millis * 1_000_000. A large value such as Long.MAX_VALUE, the usual "no limit" convention, overflowed that sum into the past, so the wait ended at once: STRICT failed the row with SCHEMA_UNAVAILABLE, AUTO silently fell back to the legacy contract, drain() returned false, and close() skipped its flush wait, which loses unacked rows in memory mode. Each wait now records its start and a saturated budget from TimeUnit.MILLISECONDS.toNanos and compares elapsed time against it, which cannot wrap. QwpSchemaCoordinator and awaitInitialSchemaMode drop their hand-rolled saturating multiplies for the same call. The Sender javadoc states that Long.MAX_VALUE waits without a limit; the builder is unchanged. testUnboundedSchemaWaitStillResolves binds the schema in STRICT and AUTO with an unlimited wait, and QwpWebSocketSenderUnboundedWaitTest holds the server ACK to prove drain() and close() keep waiting. All three fail against the previous code. 2514 client tests pass; the new class also passes on JDK 8. --- .../main/java/io/questdb/client/Sender.java | 9 +- .../qwp/client/QwpWebSocketSender.java | 30 ++-- .../sf/cursor/CursorWebSocketSendLoop.java | 3 +- .../sf/cursor/QwpSchemaCoordinator.java | 7 +- .../QwpSchemaSenderIntegrationTest.java | 22 +++ .../QwpWebSocketSenderUnboundedWaitTest.java | 146 ++++++++++++++++++ 6 files changed, 196 insertions(+), 21 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index b56551953..97547db5f 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -502,7 +502,8 @@ default Sender decimalColumn(CharSequence name, CharSequence value) { * * @param timeoutMillis upper bound on the wait; {@code <= 0} returns the * current state without blocking (the flush still - * happens before the check) + * happens before the check), and {@code Long.MAX_VALUE} + * waits without a limit * @return {@code true} if the server has acknowledged every published * frame on return, {@code false} on timeout * @throws LineSenderException if the transport has latched a terminal error @@ -2000,7 +2001,7 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { * {@link Sender#drain(long)} explicitly before close(). *

      * Set to {@code 0} or {@code -1} to opt out — close() will not wait - * at all (fast close). Pending data is then lost in memory mode and + * at all (fast close). {@code Long.MAX_VALUE} waits without a limit. Pending data is then lost in memory mode and * recovered by the next sender in SF mode. *

      * WebSocket transport only. @@ -3046,8 +3047,8 @@ public LineSenderBuilder schemaMode(SchemaMode mode) { /** * Upper bound for one schema wait: the first WebSocket upgrade, if it * has not completed yet, plus one schema lookup round trip. Reconnects - * consume the same budget; they do not restart it. Default 30 s. At the - * bound, {@link SchemaMode#AUTO} writes the row with the legacy contract + * consume the same budget; they do not restart it. Default 30 s; + * {@code Long.MAX_VALUE} waits without a limit. At the bound, {@link SchemaMode#AUTO} writes the row with the legacy contract * and {@link SchemaMode#STRICT} fails it. WebSocket transport only. */ public LineSenderBuilder schemaWaitMillis(long millis) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index b4faf3b43..cde2c9f80 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1156,14 +1156,17 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { if (timeoutMillis <= 0L) { return false; } - long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L; + // Elapsed-versus-budget, not an absolute deadline: toNanos saturates, so + // Long.MAX_VALUE waits indefinitely instead of wrapping to no wait. + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); while (cursorEngine.ackedFsn() < targetFsn) { cursorEngine.checkDurability(); if (cursorSendLoop != null) { cursorSendLoop.checkError(); } checkConnectionError(); - if (System.nanoTime() >= deadlineNanos) { + if (System.nanoTime() - startNanos >= budgetNanos) { return false; } java.util.concurrent.locks.LockSupport.parkNanos(50_000L); @@ -4160,7 +4163,8 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); } } - long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(closeFlushTimeoutMillis); restoreInterrupt[0] |= Thread.interrupted(); try { while (cursorEngine.ackedFsn() < target) { @@ -4180,7 +4184,7 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn } else { cursorSendLoop.checkError(); } - if (System.nanoTime() >= deadlineNanos) { + if (System.nanoTime() - startNanos >= budgetNanos) { long acked = cursorEngine.ackedFsn(); // Name the outage the I/O thread is riding out, when there is one. A // foreground sender now retries endpoint-policy rejections indefinitely, @@ -4460,12 +4464,13 @@ private QwpSchemaResponse resolveSchemaForCurrentTable() { if (schemaMode == Sender.SchemaMode.OFF) { return null; } - final long deadlineNanos = System.nanoTime() + schemaWaitMillis * 1_000_000L; + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(schemaWaitMillis); if (schemaMode == Sender.SchemaMode.STRICT) { - if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(deadlineNanos))) { + if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(startNanos, budgetNanos))) { return null; } - return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); + return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); } if (!cursorSendLoop.hasEverConnected()) { return null; @@ -4477,7 +4482,7 @@ private QwpSchemaResponse resolveSchemaForCurrentTable() { return null; } try { - return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(deadlineNanos)); + return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); } catch (LineSenderSchemaException e) { if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE || e.getReason() == LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) { @@ -4539,8 +4544,13 @@ private RuntimeException onRowFailure(Throwable e) { return (RuntimeException) e; } - private static long remainingSchemaMillis(long deadlineNanos) { - long remaining = deadlineNanos - System.nanoTime(); + /** + * Milliseconds left of a saturated nanosecond budget, rounded up. Comparing + * elapsed time against the budget cannot wrap, unlike an absolute deadline, + * so a budget of Long.MAX_VALUE millis means no limit. + */ + private static long remainingSchemaMillis(long startNanos, long budgetNanos) { + long remaining = budgetNanos - (System.nanoTime() - startNanos); return remaining <= 0 ? 0 : 1L + (remaining - 1L) / 1_000_000L; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 147870bd4..349b631e9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1613,8 +1613,7 @@ public boolean isWireUp() { * handshake is never interpreted as a legacy peer. */ public boolean awaitInitialSchemaMode(long timeoutMillis) { - long timeoutNanos = timeoutMillis > Long.MAX_VALUE / 1_000_000L - ? Long.MAX_VALUE : timeoutMillis * 1_000_000L; + long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); long start = System.nanoTime(); while (!hasEverConnected) { checkError(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index 9f6433c5f..417b07732 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -15,6 +15,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; @@ -39,7 +40,7 @@ QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean re + QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + " UTF-16 units"); } final String key = normalize(tableName); - final long timeoutNanos = timeoutNanos(timeoutMillis); + final long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); synchronized (this) { if (closed) { throw failure(SCHEMA_UNAVAILABLE, key, "schema coordinator is closed"); @@ -237,10 +238,6 @@ synchronized void close() { notifyAll(); } - private static long timeoutNanos(long timeoutMillis) { - return timeoutMillis > Long.MAX_VALUE / 1_000_000L ? Long.MAX_VALUE : timeoutMillis * 1_000_000L; - } - private static LineSenderSchemaException failure(LineSenderSchemaException.Reason reason, String table, String detail) { return new LineSenderSchemaException(reason, "schema lookup failed [reason=" + reason + ", table=" + table + ", detail=" + detail + ']'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 8fb7ca4c6..3c83e727f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -2238,6 +2238,28 @@ public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exceptio }); } + @Test(timeout = 10_000) + public void testUnboundedSchemaWaitStillResolves() throws Exception { + assertMemoryLeak(() -> { + for (String mode : new String[]{"strict", "auto"}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + // Long.MAX_VALUE once overflowed the absolute deadline into a zero + // wait: STRICT failed the row and AUTO fell back to legacy framing. + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + mode + ";schema_wait_millis=" + Long.MAX_VALUE + ";" + + "auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertEquals(mode, 1, handler.describeRequests.get()); + Assert.assertTrue(mode, (handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + } + }); + } + @Test(timeout = 10_000) public void testStrictSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java new file mode 100644 index 000000000..ae10a518f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java @@ -0,0 +1,146 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * {@code Long.MAX_VALUE} as a wait budget means "no limit". It once overflowed an + * absolute nanoTime deadline, so the waits below returned at once instead. + */ +public class QwpWebSocketSenderUnboundedWaitTest { + + @Test(timeout = 20_000) + public void testCloseFlushWaitsWithoutLimit() throws Exception { + assertMemoryLeak(() -> { + HeldAckHandler handler = new HeldAckHandler(); + try (TestWebSocketServer server = start(handler)) { + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;close_flush_timeout_millis=" + Long.MAX_VALUE + ";"); + sender.table("t").longColumn("v", 1).atNow(); + sender.flush(); + Assert.assertTrue(handler.received.await(5, TimeUnit.SECONDS)); + AtomicReference error = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + error.set(t); + } + }, "unbounded-close"); + closer.start(); + try { + closer.join(200); + Assert.assertTrue("close() must wait for the ACK", closer.isAlive()); + } finally { + handler.release.countDown(); + closer.join(10_000); + } + Assert.assertFalse(closer.isAlive()); + Assert.assertNull(error.get()); + Assert.assertTrue(handler.acked.get()); + } + }); + } + + @Test(timeout = 20_000) + public void testDrainWaitsWithoutLimit() throws Exception { + assertMemoryLeak(() -> { + HeldAckHandler handler = new HeldAckHandler(); + try (TestWebSocketServer server = start(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;close_flush_timeout_millis=0;")) { + sender.table("t").longColumn("v", 1).atNow(); + AtomicReference drained = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + Thread drainer = new Thread(() -> { + try { + drained.set(sender.drain(Long.MAX_VALUE)); + } catch (Throwable t) { + error.set(t); + } + }, "unbounded-drain"); + drainer.start(); + try { + Assert.assertTrue(handler.received.await(5, TimeUnit.SECONDS)); + drainer.join(200); + Assert.assertTrue("drain() must wait for the ACK", drainer.isAlive()); + } finally { + handler.release.countDown(); + drainer.join(10_000); + } + Assert.assertFalse(drainer.isAlive()); + Assert.assertNull(error.get()); + Assert.assertEquals(Boolean.TRUE, drained.get()); + } + }); + } + + private static TestWebSocketServer start(HeldAckHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + // Holds the ACK for the first data frame until the test releases it. + private static final class HeldAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicBoolean acked = new AtomicBoolean(); + private final CountDownLatch received = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private long sequence; + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + received.countDown(); + try { + if (!release.await(10, TimeUnit.SECONDS)) { + return; + } + client.sendBinary(QwpWireTestUtils.buildAck(sequence++)); + acked.set(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} From 07f8d2761e02dbfec33771320ae095efd726dd0c Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 15:07:23 +0200 Subject: [PATCH 42/51] Cache TOO_LARGE schema answers in AUTO mode QwpSchemaCoordinator evicted a TOO_LARGE DESCRIBE answer instead of caching it, so AUTO paid one blocking round trip on the producer thread for every batch of a table too wide to describe (more than 2048 columns, or a schema larger than the server's send buffer), and a pending schema-framed batch was flushed early, splitting the frame. TOO_LARGE now caches like MISSING through one isCacheable rule shared by completeResponse and applyFeedback; DENIED and UNAVAILABLE may clear at any time and are still evicted. The coordinator returns the cached TOO_LARGE response and the sender maps it in one place: AUTO writes the legacy contract, STRICT fails with UNSUPPORTED_FEATURE. An AUTO batch for an oversized table costs a map lookup, with no round trip and no exception. The negative entry stays correct because the server reports each legacy-framed table once per schema version: when the table shrinks, the next legacy ACK carries the describable schema and replaces it. STRICT writes no rows to such a table and so earns no feedback; it asks again when it finds a cached TOO_LARGE, as it did before. Coordinator tests cover caching from a response and from feedback, and that UNAVAILABLE is not cached. testTooLargeSchemaIsDescribedOnce sends five AUTO batches with one DESCRIBE (the old code sent ten) and keeps STRICT failing. 2517 client tests pass; also JDK 8. --- .../qwp/client/QwpWebSocketSender.java | 22 +++++++ .../sf/cursor/CursorWebSocketSendLoop.java | 4 +- .../sf/cursor/QwpSchemaCoordinator.java | 29 ++++++--- .../QwpSchemaSenderIntegrationTest.java | 36 +++++++++++ .../CursorWebSocketSchemaCoordinatorTest.java | 60 ++++++++++++++++++- 5 files changed, 139 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index cde2c9f80..5ac2f9000 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -50,6 +50,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; @@ -4461,6 +4462,20 @@ private QwpSchemaBinding bindingForEffectiveWrite() { * the row. Access denial still fails the row. */ private QwpSchemaResponse resolveSchemaForCurrentTable() { + QwpSchemaResponse schema = lookUpSchemaForCurrentTable(); + if (schema == null || schema.getResult() != QwpSchemaProtocol.RESULT_TOO_LARGE) { + return schema; + } + // The coordinator caches TOO_LARGE, so this costs no round trip per batch. + if (schemaMode == Sender.SchemaMode.STRICT) { + throw new LineSenderSchemaException(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + "schema lookup failed [reason=UNSUPPORTED_FEATURE, table=" + currentTableName + + ", detail=schema could not be returned within the discovery response limit]"); + } + return null; + } + + private QwpSchemaResponse lookUpSchemaForCurrentTable() { if (schemaMode == Sender.SchemaMode.OFF) { return null; } @@ -4470,6 +4485,13 @@ private QwpSchemaResponse resolveSchemaForCurrentTable() { if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(startNanos, budgetNanos))) { return null; } + // STRICT writes no rows to a table it cannot describe, so no ACK feedback + // ever reports that the table shrank: ask again instead of trusting a + // cached TOO_LARGE. AUTO keeps the cache hit; its legacy rows earn feedback. + QwpSchemaResponse cached = cursorSendLoop.peekSchema(currentTableName); + if (cached != null && cached.getResult() == QwpSchemaProtocol.RESULT_TOO_LARGE) { + return cursorSendLoop.refreshSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); + } return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); } if (!cursorSendLoop.hasEverConnected()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 349b631e9..70e57474d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1220,7 +1220,7 @@ public void checkError() { } /** - * Resolves a schema through this loop's I/O thread, returning a KNOWN or MISSING response. + * Resolves a schema through this loop's I/O thread, returning a KNOWN, MISSING or TOO_LARGE response. * This producer-side API permits one outstanding lookup. The timeout covers the entire * operation, including connection and I/O; zero permits only an existing cache hit. * Failures are reported as {@link LineSenderSchemaException}. This does not mutate rows, @@ -1252,7 +1252,7 @@ public void setAbandonSchemaLookupOnDisconnect(boolean abandon) { * Evicts the named cache entry at admission and resolves it again through the I/O thread. * This producer-side API permits one outstanding lookup. The timeout covers the entire * operation, including connection and I/O, and must leave time for a request. It returns - * a KNOWN or MISSING response and reports failures as {@link LineSenderSchemaException}. + * a KNOWN, MISSING or TOO_LARGE response and reports failures as {@link LineSenderSchemaException}. * This does not mutate rows, store-and-forward state, or acknowledgment watermarks. */ public QwpSchemaResponse refreshSchema(CharSequence tableName, long timeoutMillis) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index 417b07732..c2b815317 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -129,12 +129,12 @@ synchronized void completeResponse(WebSocketClient client, QwpSchemaResponse res complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema lookup timed out")); return; } + if (isCacheable(response.getResult())) { + put(current.key, response); + complete(current, response, null); + return; + } switch (response.getResult()) { - case QwpSchemaProtocol.RESULT_KNOWN: - case QwpSchemaProtocol.RESULT_MISSING: - put(current.key, response); - complete(current, response, null); - return; case QwpSchemaProtocol.RESULT_DENIED: remove(current.key); complete(current, null, failure(ACCESS_DENIED, current.key, "schema access denied")); @@ -143,10 +143,9 @@ synchronized void completeResponse(WebSocketClient client, QwpSchemaResponse res remove(current.key); complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema is unavailable")); return; - case QwpSchemaProtocol.RESULT_TOO_LARGE: default: remove(current.key); - complete(current, null, failure(UNSUPPORTED_FEATURE, current.key, "schema response is too large or unsupported")); + complete(current, null, failure(UNSUPPORTED_FEATURE, current.key, "unsupported schema result")); } } @@ -167,8 +166,7 @@ void applyFeedback(WebSocketResponse feedback) { if (closed) { return; } - if (schema.getResult() == QwpSchemaProtocol.RESULT_KNOWN - || schema.getResult() == QwpSchemaProtocol.RESULT_MISSING) { + if (isCacheable(schema.getResult())) { put(key, schema); } else { remove(key); @@ -238,6 +236,19 @@ synchronized void close() { notifyAll(); } + /** + * Results that stay true until the table's schema version changes, which + * the server then reports through ACK feedback. TOO_LARGE is a negative entry, + * like MISSING: without it, AUTO would pay a DESCRIBE round trip every batch + * for a table too wide to describe. DENIED and UNAVAILABLE may clear at any + * time, so they are never cached. + */ + private static boolean isCacheable(int result) { + return result == QwpSchemaProtocol.RESULT_KNOWN + || result == QwpSchemaProtocol.RESULT_MISSING + || result == QwpSchemaProtocol.RESULT_TOO_LARGE; + } + private static LineSenderSchemaException failure(LineSenderSchemaException.Reason reason, String table, String detail) { return new LineSenderSchemaException(reason, "schema lookup failed [reason=" + reason + ", table=" + table + ", detail=" + detail + ']'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 3c83e727f..2cd3ecffa 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -1818,6 +1818,42 @@ public void testAutoUnavailableSchemaUsesLegacyFrame() throws Exception { }); } + @Test + public void testTooLargeSchemaIsDescribedOnce() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_TOO_LARGE, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + // AUTO once re-sent a blocking DESCRIBE for every batch of a table + // too wide to describe; the cached negative answer now serves them. + for (int i = 0; i < 5; i++) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + } + for (byte[] frame : handler.awaitDataFrames(5)) { + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + SchemaHandler strictHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_TOO_LARGE, -1, -1); + try (TestWebSocketServer server = schemaServer(strictHandler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + for (int i = 0; i < 2; i++) { + try { + sender.table("events").stringColumn("value", "strict"); + Assert.fail("STRICT must reject a table too wide to describe"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + } + // STRICT has no legacy rows to earn feedback, so it asks again each time. + Assert.assertEquals(2, strictHandler.describeRequests.get()); + } + }); + } + @Test public void testAutoTooLargeSchemaPublishesPendingSchemaBatchBeforeLegacyRow() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java index be54b0455..be9e574b2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java @@ -86,7 +86,65 @@ public void testMissingIsCachedButRemoteFailuresAreTyped() throws Exception { } assertRemoteReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "denied"); assertRemoteReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "unavailable"); - assertRemoteReason(QwpSchemaProtocol.RESULT_TOO_LARGE, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "too-large"); + } + + @Test + public void testTooLargeIsCachedButUnavailableIsNot() throws Exception { + // A table too wide to describe stays too wide until its version changes, + // which ACK feedback reports; re-asking every batch would cost a round trip. + DescribeHandler tooLarge = new DescribeHandler(true, QwpSchemaProtocol.RESULT_TOO_LARGE); + try (TestWebSocketServer server = server(tooLarge); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("too-large").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("Wide", 5_000).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("WIDE", 0).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.peekSchema("wide").getResult()); + Assert.assertEquals(1, tooLarge.requests.get()); + } finally { + loop.close(); + } + } + DescribeHandler unavailable = new DescribeHandler(true, QwpSchemaProtocol.RESULT_UNAVAILABLE); + try (TestWebSocketServer server = server(unavailable); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("unavailable").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.resolveSchema("t", 5_000)); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(2, unavailable.requests.get()); + Assert.assertNull(loop.peekSchema("t")); + } finally { + loop.close(); + } + } + } + + @Test + public void testTooLargeFeedbackIsCached() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("too-large-feedback").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + loop.resolveSchema("seed", 5_000); + handler.sendFeedback("Wide", QwpSchemaProtocol.RESULT_TOO_LARGE); + // The marker reply follows the feedback on the wire, so the I/O thread + // has applied the feedback once the marker lookup returns. + loop.refreshSchema("marker", 5_000); + int afterMarker = handler.requests.get(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("wide", 0).getResult()); + Assert.assertEquals(afterMarker, handler.requests.get()); + } finally { + loop.close(); + } + } } @Test From 2f5bfe2f7148935c24a994a2f67e81a1033142ec Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 15:20:46 +0200 Subject: [PATCH 43/51] Document what Sender.table() can block on and throw QWP WebSocket senders resolve the table's schema eagerly in table(), which widened its contract beyond the old "select the table" javadoc. It does not connect: build() always runs ensureConnected(), so the call inside table() is a no-op for any built sender. It can, however, wait up to schema_wait_millis (with an asynchronous initial connect that includes waiting for the first connection), fail with a schema error in STRICT mode, and publish the pending batch when the table's wire contract differs, which can raise flush-timeout and backpressure errors. The javadoc now states that surface, and that a failure leaves the table selected with no row started, so calling table() again retries. No behaviour changes. --- core/src/main/java/io/questdb/client/Sender.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 97547db5f..284706727 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -766,9 +766,13 @@ default Sender shortColumn(CharSequence name, short value) { * to finalize the row. You can then start a new row by calling this method again. *
      * If you want to cancel the current row, you can call {@link #cancelRow()}. - * QWP WebSocket senders resolve the table's schema here when the server speaks schema-aware QWP, so this call may - * wait for or fail schema discovery; {@link SchemaMode} decides what happens when the schema is not obtainable. - * Setters may resolve it again when an implicitly started row crosses a batch boundary. + * QWP WebSocket senders resolve the table's schema here when the server speaks schema-aware QWP. The call may + * therefore wait up to {@code schema_wait_millis} (with an asynchronous initial connect, that includes waiting for + * the first connection), and it publishes the pending batch when this table's wire contract differs from it. So + * schema errors, and the flush-timeout and backpressure errors of that publish, can surface here, before the row + * has any values. A failure leaves the table selected and no row started; calling this method again retries the + * resolution. {@link SchemaMode} decides what happens when the schema is not obtainable. Setters may resolve it + * again when an implicitly started row crosses a batch boundary. * * @param table name of the table * @return this instance for method chaining From eac519c45eeb8d3ffd26af91211fa3063b4c5667 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 15:35:33 +0200 Subject: [PATCH 44/51] Match server NaN and Infinity decimal parsing The client's DecimalParser required NaN and Infinity to be the whole token, while the server's DecimalParser matches them as prefixes. So "NaN " or "NaNjunk" became a NULL decimal on every server ingest path, including legacy QWP text into a DECIMAL column, but failed the row in schema mode, where the client converts the text itself. isNanOrInfinite is back to the prefix match it had before this branch, so both paths store NULL. The client keeps its long exponent arithmetic and range guards: they only reject inputs such as "1e2147483647" that the server does not parse into a valid value either. DecimalParserServerCompatibilityTest now pins the prefix behaviour and that truncated tokens are still rejected, and the string-to-decimal corpus expects NULL for its NaN and Infinity junk rows. Both fail against the previous parser. 3205 client tests pass. --- .../main/java/io/questdb/client/std/DecimalParser.java | 6 ++++-- .../test/std/DecimalParserServerCompatibilityTest.java | 10 +++++++--- .../questdb/client/cutlass/qwp/string-to-decimal.tsv | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/std/DecimalParser.java b/core/src/main/java/io/questdb/client/std/DecimalParser.java index ece8c051d..e3d8aee14 100644 --- a/core/src/main/java/io/questdb/client/std/DecimalParser.java +++ b/core/src/main/java/io/questdb/client/std/DecimalParser.java @@ -338,8 +338,10 @@ private static boolean isDigit(char c) { * @return true if "NaN" or "Infinity" is found at position lo, false otherwise */ private static boolean isNanOrInfinite(char ch, CharSequence cs, int lo, int hi) { - return (ch == 'N' && hi - lo == 3 && cs.charAt(lo + 1) == 'a' && cs.charAt(lo + 2) == 'N') || - (ch == 'I' && hi - lo == 8 && cs.charAt(lo + 1) == 'n' && cs.charAt(lo + 2) == 'f' + // A prefix match, like the server's DecimalParser: "NaNjunk" parses as NULL on + // every server ingest path, so the client must not reject it. + return (ch == 'N' && lo + 2 < hi && cs.charAt(lo + 1) == 'a' && cs.charAt(lo + 2) == 'N') || + (ch == 'I' && lo + 7 < hi && cs.charAt(lo + 1) == 'n' && cs.charAt(lo + 2) == 'f' && cs.charAt(lo + 3) == 'i' && cs.charAt(lo + 4) == 'n' && cs.charAt(lo + 5) == 'i' && cs.charAt(lo + 6) == 't' && cs.charAt(lo + 7) == 'y'); } diff --git a/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java index b9ba771c0..43ab94d9c 100644 --- a/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java +++ b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java @@ -40,11 +40,15 @@ public void testPrecisionIsAtLeastOneAndZeroFitsFullScale() throws NumericExcept } @Test - public void testSpecialValuesRequireTheCompleteToken() throws NumericException { + public void testSpecialValuesMatchByPrefixLikeServer() throws NumericException { assertNullParsed("NaN", 38, 4); assertNullParsed("-Infinity", 76, 8); - assertRejected("NaNjunk", 38, 4); - assertRejected("Infinitygarbage", 76, 8); + // The server's DecimalParser treats NaN and Infinity as prefixes. + assertNullParsed("NaNjunk", 38, 4); + assertNullParsed("NaN ", 38, 4); + assertNullParsed("Infinitygarbage", 76, 8); + assertRejected("Na", 38, 4); + assertRejected("Infinit", 76, 8); } @Test diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv index d2d8231da..2fc76ec03 100644 --- a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv @@ -45,9 +45,9 @@ 128_float_then_decimal_suffix_strict_loss 1.2300fm DECIMAL128 38 2 INVALID - - - - - 128_decimal_then_float_suffix_invalid 1.2300mf DECIMAL128 38 2 INVALID - - - - - 128_nan NaN DECIMAL128 38 4 NULL - - - - NULL -128_nan_junk NaNjunk DECIMAL128 38 4 INVALID - - - - - +128_nan_junk NaNjunk DECIMAL128 38 4 NULL - - - - NULL 256_negative_infinity -Infinity DECIMAL256 76 8 NULL - - - - NULL -256_infinity_junk Infinitygarbage DECIMAL256 76 8 INVALID - - - - - +256_infinity_junk Infinitygarbage DECIMAL256 76 8 NULL - - - - NULL 256_lowercase_nan nan DECIMAL256 76 8 INVALID - - - - - 256_upper_exponent 7.89E-3 DECIMAL256 76 5 VALUE 0000000000000315 0000000000000000 0000000000000000 0000000000000000 0.00789 256_exponent_int_max 1e2147483647 DECIMAL256 76 0 INVALID - - - - - From 61457fc94e59f81a0521c8e35e1a854914b57928 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 16:04:09 +0200 Subject: [PATCH 45/51] Make schema conformance tests assert their data testUuidConformanceCorpus parsed every row but never checked what the valid rows encoded, so 8 of its 20 rows, including the case-folding row, asserted nothing. It now compares each valid row's limbs with UUID.fromString(expected); a NULL row accepts a null-bitmap entry or the both-Long.MIN_VALUE UUID null sentinel, which the server reads as NULL. Swapping the limbs in the binding now fails it. The timestamp-inputs, timestamp-units and long-to-numeric loaders had no row-count guard, so an emptied corpus or a filter that skipped every row would pass with zero cases. Each now asserts its exact count. QwpSchemaProtocolTest drops its private frame and column copies for the shared QwpSchemaTestFixtures helpers, so the frame version byte comes from QwpConstants.VERSION instead of a hardcoded 1. --- .../qwp/protocol/QwpSchemaBindingTest.java | 74 ++++++++++++++----- .../qwp/protocol/QwpSchemaProtocolTest.java | 14 +--- 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java index ee7e15a40..80c16439e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -44,6 +44,7 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.UUID; import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; @@ -59,10 +60,12 @@ public void testTimestampInputConformanceCorpus() throws Exception { Assert.assertNotNull(stream); try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { String line; + int count = 0; while ((line = lines.readLine()) != null) { if (line.startsWith("#")) { continue; } + count++; String[] fields = line.split("\t", -1); Assert.assertEquals(line, 7, fields.length); try { @@ -100,6 +103,8 @@ public void testTimestampInputConformanceCorpus() throws Exception { throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); } } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(102, count); } } @@ -179,10 +184,12 @@ public void testTimestampUnitConformanceCorpus() throws Exception { Assert.assertNotNull(stream); try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { String line; + int count = 0; while ((line = lines.readLine()) != null) { if (line.startsWith("#")) { continue; } + count++; String[] fields = line.split("\t", -1); Assert.assertEquals(line, 6, fields.length); try { @@ -210,6 +217,8 @@ public void testTimestampUnitConformanceCorpus() throws Exception { throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); } } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(48, count); } } @@ -936,10 +945,12 @@ public void testLongNumericConformanceCorpus() throws Exception { Assert.assertNotNull(stream); try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { String line; + int count = 0; while ((line = lines.readLine()) != null) { if (line.startsWith("#")) { continue; } + count++; String[] fields = line.split("\t", -1); Assert.assertEquals(line, 4, fields.length); try { @@ -975,6 +986,8 @@ public void testLongNumericConformanceCorpus() throws Exception { throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); } } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(47, count); } } @@ -1331,29 +1344,52 @@ public void testEmptyBoundEncodingContracts() throws Exception { @Test public void testUuidConformanceCorpus() throws Exception { - InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( - "/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv"); - Assert.assertNotNull(stream); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { - String line; - while ((line = reader.readLine()) != null) { - if (line.startsWith("#")) { - continue; - } - String[] fields = line.split("\t", -1); - Assert.assertEquals(line, 3, fields.length); - CharSequence input = "".equals(fields[1]) ? null : fields[1]; - try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { - QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); - if ("".equals(fields[2])) { - assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.stringColumn("u", input)); - } else { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 3, fields.length); + CharSequence input = "".equals(fields[1]) ? null : fields[1]; + String expected = fields[2]; + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + if ("".equals(expected)) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.stringColumn("u", input)); + continue; + } rows.stringColumn("u", input); - buffer.nextRow(); + buffer.nextRow(); + QwpTableBuffer.ColumnBuffer column = buffer.getColumnCount() == 0 ? null : buffer.getColumn(0); + if ("".equals(expected)) { + // NULL is either a null-bitmap entry or the UUID null sentinel + // (both limbs Long.MIN_VALUE), which the server reads as NULL. + Assert.assertTrue(fields[0], column == null || column.isNull(0) + || (Unsafe.getUnsafe().getLong(column.getDataAddress()) == Long.MIN_VALUE + && Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES) == Long.MIN_VALUE)); + } else { + UUID uuid = UUID.fromString(expected); + Assert.assertNotNull(fields[0], column); + Assert.assertFalse(fields[0], column.isNull(0)); + // The buffer stores the low limb first, as on the wire. + Assert.assertEquals(fields[0], uuid.getLeastSignificantBits(), + Unsafe.getUnsafe().getLong(column.getDataAddress())); + Assert.assertEquals(fields[0], uuid.getMostSignificantBits(), + Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES)); + } } } } - } + Assert.assertEquals(20, count); + }); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java index bbfea31ba..ef1aad876 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java @@ -36,6 +36,9 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.frame; + public class QwpSchemaProtocolTest { @Test public void testAcceptsMaximumResponseColumnNameBoundaries() { @@ -260,10 +263,6 @@ private static void assertMalformed(byte[] frame, boolean peek) { } } - private static byte[] column(String name, int type, byte[] params) { - return columnBytes(name.getBytes(StandardCharsets.UTF_8), type, params); - } - private static byte[] columnBytes(byte[] name, int type, byte[] params) { ByteBuffer b = ByteBuffer.allocate(2 + name.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN); b.putShort((short) name.length).put(name).putInt(type).putShort((short) params.length).put(params); @@ -294,13 +293,6 @@ private static long peek(byte[] frame) { } } - private static byte[] frame(byte[] payload) { - ByteBuffer b = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN); - b.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) - .putShort((short) 0).putInt(payload.length).put(payload); - return b.array(); - } - private static byte[] knownResponse(long requestId, int tableId, long version, int designated, byte[]... columns) { int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; for (byte[] column : columns) length += column.length; From c36a5072ec7af104083fcb7031c7f4d994853e72 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Wed, 23 Sep 2026 17:53:47 +0200 Subject: [PATCH 46/51] Simplify the QWP schema coordinator cache The schema cache was a LinkedHashMap capped at 1_000_000 entries with an eviction loop. The map has no access order, so the eviction was FIFO and would have dropped the hottest tables first; in practice it never fired, because the cache is bounded by the tables the sender writes, like its per-table buffers, each far heavier than an entry. It is now a final HashMap without the cap; lookups allocate nothing. With the map always present, the four null checks, the lazy creation and the put/remove helpers go away, and completeResponse evicts once before its failure switch instead of in each branch. Request ids are now assigned only in requestToSend. resolve() used to check id exhaustion and encode the DESCRIBE itself, duplicating the path requestToSend already ran after a reconnect cleared the id. Every send attempt, first or after a reconnect, now takes a fresh id there. An exhausted id space (2^63 lookups) now fails the lookup at send time with the same error. 2517 QWP client tests, the coordinator tests on JDK 8 and the OSS QWP suite (1745 tests) pass. --- .../sf/cursor/QwpSchemaCoordinator.java | 60 ++++++------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index c2b815317..e2c6609d5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -12,9 +12,7 @@ import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.std.Chars; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; +import java.util.HashMap; import java.util.concurrent.TimeUnit; import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; @@ -22,8 +20,9 @@ import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; final class QwpSchemaCoordinator { - static final int MAX_CACHE_ENTRIES = 1_000_000; - private LinkedHashMap cache; + // Bounded by the tables this sender writes, like its per-table buffers, + // each far heavier than an entry, so it needs no eviction. + private final HashMap cache = new HashMap<>(); private volatile boolean closed; private volatile boolean hasPendingRequest; private long nextRequestId = 1; @@ -46,12 +45,12 @@ QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean re throw failure(SCHEMA_UNAVAILABLE, key, "schema coordinator is closed"); } if (!refresh) { - QwpSchemaResponse cached = cache == null ? null : cache.get(key); + QwpSchemaResponse cached = cache.get(key); if (cached != null) { return cached; } } else { - remove(key); + cache.remove(key); } if (timeoutNanos - (System.nanoTime() - startNanos) <= 0) { throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); @@ -59,13 +58,7 @@ QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean re if (request != null) { throw failure(SCHEMA_UNAVAILABLE, key, "another schema lookup is already in progress"); } - if (nextRequestId <= 0 || nextRequestId == Long.MAX_VALUE) { - throw failure(UNSUPPORTED_FEATURE, key, "schema request id space is exhausted"); - } - long id = nextRequestId; - byte[] message = QwpSchemaProtocol.encodeDescribe(id, tableName); - nextRequestId++; - Request own = new Request(id, key, tableName.toString(), startNanos, timeoutNanos, message); + Request own = new Request(key, tableName.toString(), startNanos, timeoutNanos); request = own; // The I/O loop polls hasPendingRequest on every pass; waking it would // cut its reconnect backoff short while the wire is down. @@ -97,6 +90,8 @@ synchronized Request requestToSend(WebSocketClient client) { complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema lookup timed out")); return null; } + // Every send attempt, first or after a reconnect, takes a fresh id, so a + // reply to an earlier attempt can never match this one. if (request.id == 0) { if (nextRequestId <= 0 || nextRequestId == Long.MAX_VALUE) { complete(request, null, failure(UNSUPPORTED_FEATURE, request.key, "schema request id space is exhausted")); @@ -130,21 +125,19 @@ synchronized void completeResponse(WebSocketClient client, QwpSchemaResponse res return; } if (isCacheable(response.getResult())) { - put(current.key, response); + cache.put(current.key, response); complete(current, response, null); return; } + cache.remove(current.key); switch (response.getResult()) { case QwpSchemaProtocol.RESULT_DENIED: - remove(current.key); complete(current, null, failure(ACCESS_DENIED, current.key, "schema access denied")); return; case QwpSchemaProtocol.RESULT_UNAVAILABLE: - remove(current.key); complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema is unavailable")); return; default: - remove(current.key); complete(current, null, failure(UNSUPPORTED_FEATURE, current.key, "unsupported schema result")); } } @@ -167,9 +160,9 @@ void applyFeedback(WebSocketResponse feedback) { return; } if (isCacheable(schema.getResult())) { - put(key, schema); + cache.put(key, schema); } else { - remove(key); + cache.remove(key); } } } @@ -203,7 +196,7 @@ synchronized void connectionLost(WebSocketClient client, boolean abandon) { * or {@code null}. Never sends a request and never waits. */ synchronized QwpSchemaResponse peek(CharSequence tableName) { - if (closed || cache == null) { + if (closed) { return null; } return cache.get(normalize(tableName)); @@ -223,13 +216,13 @@ synchronized boolean invalidResponse(WebSocketClient client, long requestId) { synchronized void clearCache() { if (!closed) { - cache = null; + cache.clear(); } } synchronized void close() { closed = true; - cache = null; + cache.clear(); if (request != null) { complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema coordinator is closed")); } @@ -284,23 +277,6 @@ private void complete(Request own, QwpSchemaResponse response, LineSenderSchemaE notifyAll(); } - private void put(String key, QwpSchemaResponse schema) { - if (cache == null) { - cache = new LinkedHashMap<>(); - } - cache.put(key, schema); - while (cache.size() > MAX_CACHE_ENTRIES) { - Iterator> iterator = cache.entrySet().iterator(); - iterator.next(); - iterator.remove(); - } - } - - private void remove(String key) { - if (cache != null) { - cache.remove(key); - } - } static final class Request { final long startNanos; @@ -314,13 +290,11 @@ static final class Request { QwpSchemaResponse response; WebSocketClient sentClient; - Request(long id, String key, String tableName, long startNanos, long timeoutNanos, byte[] message) { - this.id = id; + Request(String key, String tableName, long startNanos, long timeoutNanos) { this.key = key; this.tableName = tableName; this.startNanos = startNanos; this.timeoutNanos = timeoutNanos; - this.message = message; } long remainingNanos() { From 6066eebfefe4b82f2e7ec0db5e5fd88a519dd920 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 24 Sep 2026 13:19:59 +0200 Subject: [PATCH 47/51] Keep QWP schema cache across reconnects The send loop cleared the schema cache on every reconnect, so the next batch for each table paid a blocking DESCRIBE round trip, one table at a time. The server ingests by column name and uses the frame's schema identity only to decide on feedback, and a new connection reports feedback for every stale table on its first ACK. The cache now survives the reconnect. The clear was also the only way out of one stuck state. When a column changes type, a row that the stale snapshot rejects never reaches the server, so no feedback refreshes the snapshot. QwpSchemaBinding now records when the snapshot type of a known column rejects a value: an unsupported conversion, an out-of-range value, excess decimal precision or an array rank mismatch. The sender's row-failure path then evicts that snapshot if the cache still holds the same identity, so the next row looks the table up again. Inputs that no column type can take, invalid names, nulls and malformed text never evict. Tests cover a reconnect that reuses the cache without a DESCRIBE, and the lookup that follows a type rejection. --- .../qwp/client/QwpWebSocketSender.java | 16 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 14 ++- .../sf/cursor/QwpSchemaCoordinator.java | 19 +++ .../qwp/protocol/QwpSchemaBinding.java | 116 +++++++++++++----- .../QwpSchemaSenderIntegrationTest.java | 67 ++++++++-- .../CursorWebSocketSchemaLifecycleTest.java | 8 +- 6 files changed, 191 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 5ac2f9000..e64267104 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4412,7 +4412,8 @@ private void ensureConnected() { * A table pins one schema snapshot per batch. Once the table holds pending rows, * later rows of the same batch validate against that snapshot without consulting * the cache; the first row after a flush or reset adopts whatever the cache holds - * by then, which ACK schema feedback and reconnects keep current. + * by then, which ACK schema feedback keeps current. A rejection that a column + * type change could explain evicts the snapshot; see {@link #onRowFailure}. *

      * When no snapshot is pinned, the configured {@link Sender.SchemaMode} decides * between a schema lookup and the legacy contract; see @@ -4557,8 +4558,19 @@ private void unbindCurrentTableBuffer() { // Single home of the row-failure policy: roll the whole row back, then report // the failure as raised. A schema rejection is final for the batch's snapshot; - // the sender never looks up fresh metadata from a failed setter. + // the sender never looks up fresh metadata from a failed setter. When the + // snapshot type of a known column rejected the value, the column may have + // changed type since, and no frame may ever reach the server to earn feedback. + // The sender then evicts the snapshot, so the next batch looks the table up. private RuntimeException onRowFailure(Throwable e) { + QwpSchemaBinding binding = currentTableBuffer != null ? currentTableBuffer.getSchemaBinding() : null; + if (binding != null && binding.hasTargetTypeConflict()) { + binding.clearTargetTypeConflict(); + CursorWebSocketSendLoop loop = cursorSendLoop; + if (loop != null) { + loop.evictSchema(currentTableBuffer.getTableName(), binding.getTableId(), binding.getMetadataVersion()); + } + } rollbackRow(); if (e instanceof Error) { throw (Error) e; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 70e57474d..dd984e464 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1239,6 +1239,14 @@ public QwpSchemaResponse peekSchema(CharSequence tableName) { return schemaCoordinator.peek(tableName); } + /** + * Evicts the cached KNOWN snapshot of {@code tableName} if it still has the + * given identity, so the next batch that writes the table looks it up again. + */ + public void evictSchema(CharSequence tableName, int tableId, long metadataVersion) { + schemaCoordinator.evictKnown(tableName, tableId, metadataVersion); + } + /** * Selects what happens to a schema lookup in flight when the connection * drops: {@code true} fails it immediately, {@code false} (the default) keeps @@ -2783,7 +2791,11 @@ private void swapClient(WebSocketClient newClient) { } WebSocketClient old = this.client; this.client = newClient; - schemaCoordinator.clearCache(); + // The schema cache survives the reconnect. The server ingests by column name + // and ignores the identity a frame carries, except to decide on feedback. A + // new connection has reported nothing yet, so the first ACK for each table + // refreshes any snapshot that went stale during the outage. Clearing here + // would cost one blocking DESCRIBE per table on the next batch. this.lastReconnectError = null; if (old != null) { try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index e2c6609d5..db35c4d87 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -220,6 +220,25 @@ synchronized void clearCache() { } } + /** + * Drops the cached KNOWN snapshot of {@code tableName} when it still has the + * given identity. A newer snapshot, such as one that ACK feedback installed + * after the caller pinned its binding, stays cached. + */ + synchronized void evictKnown(CharSequence tableName, int tableId, long metadataVersion) { + if (closed) { + return; + } + String key = normalize(tableName); + QwpSchemaResponse cached = cache.get(key); + if (cached != null + && cached.getResult() == QwpSchemaProtocol.RESULT_KNOWN + && cached.getTableId() == tableId + && cached.getMetadataVersion() == metadataVersion) { + cache.remove(key); + } + } + synchronized void close() { closed = true; cache.clear(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java index d40aed680..d46c10302 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -79,6 +79,7 @@ public final class QwpSchemaBinding { private final QwpTableBuffer buffer; private final QwpSchemaResponse schema; private final String tableName; + private boolean hasTargetTypeConflict; private Decimal256 decimalTextScratch; private StringSink charTextSink; private StringSink floatingTextSink; @@ -171,7 +172,7 @@ public QwpSchemaBinding decimalColumn(CharSequence name, CharSequence value, Dec } if (!ColumnType.isDecimal(targetType) && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "DECIMAL256", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "DECIMAL256", targetType, "conversion is not implemented"); } parseDecimalText(name, value, scratch, targetType); if (scratch.isNull()) { @@ -249,7 +250,7 @@ public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { column.addString(value ? "true" : "false"); break; default: - throw unsupported(name, "BOOLEAN", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "BOOLEAN", targetType, "conversion is not implemented"); } return this; } @@ -401,7 +402,7 @@ public QwpSchemaBinding geoHashColumn(CharSequence name, long bits, int precisio } else if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { column.addString(formatGeoHash(value, precisionBits)); } else { - throw unsupported(name, "GEOHASH", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "GEOHASH", targetType, "conversion is not implemented"); } return this; } @@ -445,7 +446,7 @@ public QwpSchemaBinding ipv4Column(CharSequence name, int address) { if (targetType != ColumnType.IPv4 && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "IPv4", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "IPv4", targetType, "conversion is not implemented"); } if (address == Numbers.IPv4_NULL) { column.addNull(); @@ -472,7 +473,7 @@ public QwpSchemaBinding ipv4Column(CharSequence name, CharSequence address) { if (targetType != ColumnType.IPv4 && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "IPv4", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "IPv4", targetType, "conversion is not implemented"); } if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { throw error(INVALID_VALUE, name, "IPv4", targetType, @@ -509,7 +510,7 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { && targetType != ColumnType.DATE && targetType != ColumnType.TIMESTAMP_MICRO && targetType != ColumnType.TIMESTAMP_NANO) { - throw unsupported(name, "LONG", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "LONG", targetType, "conversion is not implemented"); } if (ColumnType.isDecimal(targetType)) { appendIntegerDecimal(column, name, value, "LONG", targetType, value == Long.MIN_VALUE); @@ -558,7 +559,7 @@ public QwpSchemaBinding longColumn(CharSequence name, long value) { column.addSymbol(formatLong(value)); break; default: - throw unsupported(name, "LONG", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "LONG", targetType, "conversion is not implemented"); } return this; } @@ -575,7 +576,7 @@ public QwpSchemaBinding long256Column(CharSequence name, long l0, long l1, long if (targetType != ColumnType.LONG256 && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "LONG256", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "LONG256", targetType, "conversion is not implemented"); } if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { @@ -620,7 +621,7 @@ public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { } try { if (!column.addSchemaStringDecimal(value, targetPrecision, targetScale)) { - throw error(INVALID_VALUE, name, "STRING", targetType, "decimal value exceeds target precision"); + throw targetValueConflict(name, "STRING", targetType, "decimal value exceeds target precision"); } } catch (NumericException e) { throw error(INVALID_VALUE, name, "STRING", targetType, "invalid decimal text"); @@ -688,7 +689,7 @@ public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { appendStringLong256(column, name, value); break; default: - throw unsupported(name, "STRING", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "STRING", targetType, "conversion is not implemented"); } } return this; @@ -722,7 +723,7 @@ public QwpSchemaBinding charColumn(CharSequence name, char value) { column.addString(sink); break; default: - throw unsupported(name, "CHAR", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "CHAR", targetType, "conversion is not implemented"); } return this; } @@ -745,7 +746,7 @@ public QwpSchemaBinding symbol(CharSequence name, CharSequence value) { column.addSymbol(value); break; default: - throw unsupported(name, "SYMBOL", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "SYMBOL", targetType, "conversion is not implemented"); } return this; } @@ -771,7 +772,7 @@ public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUni && targetType != ColumnType.TIMESTAMP_NANO && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "TIMESTAMP", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "TIMESTAMP", targetType, "conversion is not implemented"); } if (unit == null) { throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp unit is null"); @@ -785,7 +786,7 @@ public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUni ? NanosTimestampDriver.INSTANCE.from(value, unit) : MicrosTimestampDriver.INSTANCE.from(value, unit); } catch (ArithmeticException e) { - throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + throw targetValueConflict(name, "TIMESTAMP", targetType, "value is outside target timestamp range"); } if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { appendTimestampText(column, converted); @@ -815,7 +816,7 @@ public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { && targetType != ColumnType.TIMESTAMP_NANO && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "TIMESTAMP", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "TIMESTAMP", targetType, "conversion is not implemented"); } if (value == null) { throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp value is null"); @@ -826,7 +827,7 @@ public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { ? instantTimestamp(value, 1_000_000_000L, value.getNano()) : instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L); } catch (ArithmeticException e) { - throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + throw targetValueConflict(name, "TIMESTAMP", targetType, "value is outside target timestamp range"); } if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { appendTimestampText(column, converted); @@ -850,7 +851,7 @@ public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { return this; } if (targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { - throw unsupported(name, "UUID", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "UUID", targetType, "conversion is not implemented"); } if (lo == Long.MIN_VALUE && hi == Long.MIN_VALUE) { column.addNull(); @@ -909,6 +910,24 @@ public long getMetadataVersion() { return schema.getMetadataVersion(); } + /** + * Clears the conflict that {@link #hasTargetTypeConflict()} reports, once the + * caller has acted on it. + */ + public void clearTargetTypeConflict() { + hasTargetTypeConflict = false; + } + + /** + * Reports whether the last rejected value targeted a schema column whose + * snapshot type cannot take it. A column type change after the snapshot + * causes exactly this, so the caller evicts the snapshot from its cache. + * Input families that no column type can take never set it. + */ + public boolean hasTargetTypeConflict() { + return hasTargetTypeConflict; + } + private LineSenderSchemaException error( LineSenderSchemaException.Reason reason, CharSequence column, @@ -931,6 +950,14 @@ private LineSenderSchemaException error( return new LineSenderSchemaException(reason, message); } + // Only a column the snapshot knows records the conflict; an inferred column's + // type comes from this batch, so a fresh snapshot could not change the outcome. + private void recordTargetTypeConflict(CharSequence name) { + if (name != null && schema.getColumnIndex(name) >= 0) { + hasTargetTypeConflict = true; + } + } + private void requireUsableSchema() { switch (schema.getResult()) { case QwpSchemaProtocol.RESULT_KNOWN: @@ -998,7 +1025,7 @@ private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String input int targetType = targetType(index, expectedTargetType); QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); if (column != null && targetType != expectedTargetType) { - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } return column; } @@ -1022,15 +1049,15 @@ private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn( } if (!ColumnType.isArray(targetType) || ColumnType.decodeArrayElementType(targetType) != ColumnType.DOUBLE) { - throw unsupported(name, "DOUBLE_ARRAY", targetType, "conversion is not implemented"); + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "conversion is not implemented"); } int targetDimensions = ColumnType.decodeWeakArrayDimensionality(targetType); if (targetDimensions < 1 || targetType != ColumnType.encodeArrayType(ColumnType.DOUBLE, targetDimensions)) { - throw unsupported(name, "DOUBLE_ARRAY", targetType, "invalid target array type"); + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "invalid target array type"); } if (targetDimensions != sourceDimensions) { - throw unsupported(name, "DOUBLE_ARRAY", targetType, + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "array dimensionality mismatch [sourceDims=" + sourceDimensions + ", targetDims=" + targetDimensions + ']'); } @@ -1063,7 +1090,7 @@ private QwpTableBuffer.ColumnBuffer targetColumn( return null; } if (wireType == 0) { - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } return claimInferred(existing, name, wireType); } @@ -1077,7 +1104,7 @@ private QwpTableBuffer.ColumnBuffer targetColumn( } final byte wireType = wireType(targetType); if (wireType == 0) { - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } if (schema.hasColumnExtensionParameters(index)) { throw unsupported(name, inputType, targetType, "parameterized target type"); @@ -1227,10 +1254,10 @@ private QwpSchemaBinding appendDecimal( int targetScale = ColumnType.getDecimalScale(targetType); try { if (!column.addSchemaDecimal(value, targetPrecision, targetScale)) { - throw error(INVALID_VALUE, name, inputType, targetType, "decimal value exceeds target precision"); + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); } } catch (NumericException e) { - throw error(INVALID_VALUE, name, inputType, targetType, "decimal value cannot be rescaled exactly"); + throw targetValueConflict(name, inputType, targetType, "decimal value cannot be rescaled exactly"); } return this; } @@ -1259,11 +1286,11 @@ private QwpSchemaBinding decimalNonDecimalColumn( column.addString(sink); return this; } - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } private LineSenderSchemaException invalidRange(CharSequence name, String inputType, int targetType) { - return error(INVALID_VALUE, name, inputType, targetType, "value is outside target type range"); + return targetValueConflict(name, inputType, targetType, "value is outside target type range"); } private void parseDecimalText(CharSequence name, CharSequence value, Decimal256 scratch, int targetType) { @@ -1296,7 +1323,7 @@ private QwpSchemaBinding integerColumn( && targetType != ColumnType.DATE && targetType != ColumnType.TIMESTAMP_MICRO && targetType != ColumnType.TIMESTAMP_NANO) { - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } if (ColumnType.isDecimal(targetType)) { appendIntegerDecimal(column, name, value, inputType, targetType, sourceNull); @@ -1370,10 +1397,10 @@ private void appendIntegerDecimal( } try { if (!column.addSchemaLongDecimal(value, targetPrecision, targetScale)) { - throw error(INVALID_VALUE, name, inputType, targetType, "decimal value exceeds target precision"); + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); } } catch (NumericException e) { - throw error(INVALID_VALUE, name, inputType, targetType, "decimal value cannot be rescaled exactly"); + throw targetValueConflict(name, inputType, targetType, "decimal value cannot be rescaled exactly"); } } @@ -1778,7 +1805,7 @@ private QwpSchemaBinding floatingNonNumericColumn( } try { if (!column.addSchemaStringDecimal(formatFloating(value), targetPrecision, targetScale)) { - throw error(INVALID_VALUE, name, inputType, targetType, "decimal value exceeds target precision"); + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); } } catch (NumericException e) { throw error(INVALID_VALUE, name, inputType, targetType, @@ -1787,7 +1814,7 @@ private QwpSchemaBinding floatingNonNumericColumn( return this; } if (!isTextTarget(targetType)) { - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } if (Double.isNaN(value)) { column.addNull(); @@ -1802,7 +1829,7 @@ private QwpSchemaBinding floatingNonNumericColumn( column.addSymbol(formatFloating(value)); break; default: - throw unsupported(name, inputType, targetType, "conversion is not implemented"); + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); } return this; } @@ -1909,6 +1936,29 @@ private int targetType(int index, int inferredType) { return index >= 0 ? schema.getColumnType(index) : inferredType; } + // Rejects an input that the snapshot type of the named column cannot take. + private LineSenderSchemaException targetTypeConflict( + CharSequence name, + String inputType, + int targetType, + String detail + ) { + recordTargetTypeConflict(name); + return unsupported(name, inputType, targetType, detail); + } + + // Rejects a value that the snapshot type of the named column cannot hold, such + // as an INT overflow after the server widened the column to LONG. + private LineSenderSchemaException targetValueConflict( + CharSequence name, + String inputType, + int targetType, + String detail + ) { + recordTargetTypeConflict(name); + return error(INVALID_VALUE, name, inputType, targetType, detail); + } + private LineSenderSchemaException unsupported(CharSequence name, String inputType, int targetType, String detail) { return error(UNSUPPORTED_FEATURE, name, inputType, targetType, detail); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java index 2cd3ecffa..37817e9ea 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -1967,20 +1967,12 @@ public void testTableNegotiatesBeforeNullNoops() throws Exception { } @Test - public void testUnsupportedSetterCannotBypassSchemaAndInvalidNameDoesNotRefresh() throws Exception { + public void testUnsupportedSetterEvictsSnapshotAndInvalidNameDoesNotRefresh() throws Exception { assertMemoryLeak(() -> { SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 2, 3); try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { sender.table("events"); - try { - sender.longColumn("id", 1); - Assert.fail("expected UUID target rejection"); - } catch (LineSenderSchemaException e) { - Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); - } - Assert.assertEquals(1, handler.describeRequests.get()); - sender.table("events"); try { sender.stringColumn("bad\nname", "x"); Assert.fail("expected invalid column name"); @@ -1988,6 +1980,20 @@ public void testUnsupportedSetterCannotBypassSchemaAndInvalidNameDoesNotRefresh( Assert.assertTrue(e.getMessage().contains("column name")); } Assert.assertEquals(1, handler.describeRequests.get()); + sender.table("events"); + try { + sender.longColumn("id", 1); + Assert.fail("expected UUID target rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + // The snapshot type of a known column rejected the value, which a type + // change could explain, so the rejection evicted the snapshot and the + // next row looks the table up again. + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertEquals(2, handler.describeRequests.get()); } }); } @@ -2491,6 +2497,49 @@ public void testAutoOutageWritesUnseenTableLegacyAndNextBatchAdoptsSchema() thro }); } + @Test(timeout = 10_000) + public void testReconnectKeepsCachedSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + new FrameReader(first.awaitDataFrame()).schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(1, first.describeRequests.get()); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + // The next batch binds from the snapshot cached before the outage, so + // the reconnect costs no DESCRIBE round trip. + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + FrameReader frame = new FrameReader(second.awaitDataFrame()); + frame.schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, frame.u8()); + Assert.assertEquals(3, frame.i64()); + Assert.assertEquals(4, frame.i64()); + frame.eof(); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + @Test(timeout = 10_000) public void testAutoOutagePublishesPendingSchemaBatchBeforeLegacyRow() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java index 19d1f6484..b47f3da82 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -143,11 +143,11 @@ public void testDisconnectRetriesCurrentLookupRetainsCacheAndRejectsStaleReply() Assert.assertSame(lost.response.get(), loop.resolveSchema("lost", 0)); Assert.assertEquals(3, server.handshakeCount()); + // The cache survives the reconnect. ACK feedback on the new + // connection refreshes any entry that went stale meanwhile. int requestsBeforeCacheProbe = handler.requests; - QwpSchemaResponse replacement = loop.resolveSchema("cached", 5_000); - Assert.assertTrue(replacement.getRequestId() > 0); - Assert.assertNotSame(cached, replacement); - Assert.assertEquals(requestsBeforeCacheProbe + 1, handler.requests); + Assert.assertSame(cached, loop.resolveSchema("cached", 5_000)); + Assert.assertEquals(requestsBeforeCacheProbe, handler.requests); } finally { reconnect.allow.countDown(); } From 06653258ff12af5ee0071ffbf782d7df0a79e317 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Thu, 24 Sep 2026 15:56:22 +0200 Subject: [PATCH 48/51] fix(client): reject unpaired surrogates in table names --- .../java/io/questdb/client/cairo/TableUtils.java | 9 +++++++++ .../cutlass/qwp/protocol/QwpSchemaProtocol.java | 10 ---------- .../client/test/cairo/TableUtilsTest.java | 5 +++++ .../qwp/client/QwpWebSocketSenderTest.java | 16 ++++++++++++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cairo/TableUtils.java b/core/src/main/java/io/questdb/client/cairo/TableUtils.java index f250e9f80..6b13bea3b 100644 --- a/core/src/main/java/io/questdb/client/cairo/TableUtils.java +++ b/core/src/main/java/io/questdb/client/cairo/TableUtils.java @@ -87,6 +87,15 @@ public static boolean isValidTableName(CharSequence tableName, int fsFileNameLim for (int i = 0; i < length; i++) { char c = tableName.charAt(i); + if (Character.isHighSurrogate(c)) { + if (++i >= length || !Character.isLowSurrogate(tableName.charAt(i))) { + return false; + } + continue; + } + if (Character.isLowSurrogate(c)) { + return false; + } switch (c) { case '.': if (i == 0 || i == length - 1 || tableName.charAt(i - 1) == '.') { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java index 92c31129b..a9e35dd57 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java @@ -237,16 +237,6 @@ private static byte[] encodeName(CharSequence value, String what) { if (value == null || value.length() == 0 || value.length() > MAX_NAME_UTF16_LENGTH) { throw new IllegalArgumentException(what + " must contain 1 to " + MAX_NAME_UTF16_LENGTH + " UTF-16 units"); } - for (int i = 0; i < value.length(); i++) { - char c = value.charAt(i); - if (Character.isHighSurrogate(c)) { - if (++i >= value.length() || !Character.isLowSurrogate(value.charAt(i))) { - throw new IllegalArgumentException(what + " contains an unpaired surrogate"); - } - } else if (Character.isLowSurrogate(c)) { - throw new IllegalArgumentException(what + " contains an unpaired surrogate"); - } - } byte[] bytes = value.toString().getBytes(StandardCharsets.UTF_8); if (bytes.length > MAX_NAME_UTF8_LENGTH) { throw new IllegalArgumentException(what + " exceeds " + MAX_NAME_UTF8_LENGTH + " UTF-8 bytes"); diff --git a/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java b/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java index f38ebf4d2..e1e209afc 100644 --- a/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java +++ b/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java @@ -82,8 +82,13 @@ public void testIsValidTableName() { Assert.assertFalse(TableUtils.isValidTableName("\\..", 127)); Assert.assertFalse(TableUtils.isValidTableName("/..", 127)); Assert.assertFalse(TableUtils.isValidTableName("../", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uFEFFt", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uD800t", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uDB80t", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uDFFFt", 127)); Assert.assertTrue(TableUtils.isValidTableName("table name", 127)); + Assert.assertTrue(TableUtils.isValidTableName("t\uD83D\uDE00t", 127)); Assert.assertTrue(TableUtils.isValidTableName("table name.csv", 127)); Assert.assertTrue(TableUtils.isValidTableName("table-name", 127)); Assert.assertTrue(TableUtils.isValidTableName("table_name", 127)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index cc35038ec..f823bcc1b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -864,6 +864,22 @@ public void testIllegalTableNameIsEscapedInTheMessage() throws Exception { }); } + @Test + public void testUnpairedSurrogateTableNameIsRejectedBeforeConnect() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + for (char invalidChar : new char[]{'\uD800', '\uDB80', '\uDFFF'}) { + try { + sender.table("bad" + invalidChar + "name"); + Assert.fail("an unpaired surrogate must be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table name contains illegal characters")); + } + } + } + }); + } + private static MicrobatchBuffer getMicrobatchBuffer(QwpWebSocketSender sender, String fieldName) throws Exception { Field field = QwpWebSocketSender.class.getDeclaredField(fieldName); field.setAccessible(true); From cfe89a7817440090583f556dfd129b287de152d4 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 25 Sep 2026 13:57:23 +0200 Subject: [PATCH 49/51] Retire aborted schema tail before first connect A store-and-forward slot recovered with only a deferred (uncommitted) tail of schema-framed frames made requiresSchema() true. The foreground sender set that requirement on its reconnect factory before any send loop existed, so build() failed with QwpSchemaCapabilityMismatchException against every server without schema support, even with schema_mode=off. The tail belongs to an aborted transaction and never reaches the wire. ensureConnected() now calls retireRecoveredOrphanTailIfReady() before it sets the schema requirement, matching BackgroundDrainer. The sender starts against a legacy server and sends new rows as legacy frames. QwpSchemaReplayNetworkTest now asserts that outcome instead of the startup failure. --- .../qwp/client/QwpWebSocketSender.java | 5 ++++ .../sf/cursor/QwpSchemaReplayNetworkTest.java | 28 ++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index e64267104..35219b687 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4265,6 +4265,11 @@ private void ensureConnected() { connectionDispatcher = new SenderConnectionDispatcher( connectionListener, connectionListenerInboxCapacity); } + // A recovered deferred-only tail is an aborted transaction that never + // reaches the wire. Retire it locally before the capability check, as + // BackgroundDrainer does, so its schema frames cannot demand a + // schema-capable peer and block startup against a legacy server. + cursorEngine.retireRecoveredOrphanTailIfReady(); CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory(); reconnectFactory.setSchemaRequired(cursorEngine.requiresSchema()); switch (initialConnectMode) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java index a8df48295..d59ddd8f4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -295,21 +295,35 @@ public void testPublicSenderSchemaFrameSurvivesRestartByteForByte() throws Excep } @Test - public void testDeferredOnlySchemaTailIsRetainedByOldPeerThenAbortedLocally() throws Exception { - File root = temp.newFolder("deferred"); + public void testDeferredOnlySchemaTailIsAbortedLocallyBeforeConnectingToOldPeer() throws Exception { + File root = temp.newFolder("deferred-old-peer"); String slot = new File(root, "default").getAbsolutePath(); seedDeferredSchemaTail(slot); - RecordingHandler oldHandler = new RecordingHandler(false, "deferred"); + RecordingHandler oldHandler = new RecordingHandler(true, "deferred"); try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { oldPeer.start(); Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertThrows(QwpSchemaCapabilityMismatchException.class, - () -> Sender.fromConfig(config(oldPeer.getPort(), root, false))); - Assert.assertEquals(0, oldHandler.frames.size()); + try (Sender sender = Sender.fromConfig(config(oldPeer.getPort(), root, false))) { + sender.table("deferred").longColumn("n", 2).atNow(); + sender.flush(); + Assert.assertTrue(sender.drain(10_000)); + } + Assert.assertEquals("only the new legacy row may reach the old peer", 1, oldHandler.frames.size()); + Assert.assertEquals(0, oldHandler.frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + } + try (CursorSendEngine recovered = new CursorSendEngine(slot, SEGMENT_SIZE)) { + Assert.assertFalse(recovered.requiresSchema()); + Assert.assertEquals(recovered.publishedFsn(), recovered.ackedFsn()); } - assertRetained(slot, 0); assertNoQuarantine(root); + } + + @Test + public void testDeferredOnlySchemaTailIsAbortedLocallyBeforeConnectingToSupportingPeer() throws Exception { + File root = temp.newFolder("deferred"); + String slot = new File(root, "default").getAbsolutePath(); + seedDeferredSchemaTail(slot); RecordingHandler supportingHandler = new RecordingHandler(true, "deferred"); try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { From 89423e63bade9f66460756a5b9f963f4a558dad0 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 25 Sep 2026 15:28:40 +0200 Subject: [PATCH 50/51] Stop allocating on schema cache hits QwpSchemaCoordinator.resolve() lower-cased the table name into a new String before every cache lookup, hit or miss. In AUTO and STRICT modes the sender resolves twice on the first row of each table in each batch, so a schema-aware sender produced two garbage Strings per table per batch on the producer thread, and one pair per row for callers that flush every row. QwpTableBuffer now computes the lower-cased schema key once, reusing the name itself when it is already lower case. The sender passes that key to resolveSchema(), peekSchema() and refreshSchema(), and normalize() returns an already lower-case String unchanged, so a cache hit no longer allocates. DESCRIBE requests now carry the lower-cased name; the server resolves table names case-insensitively, so the answer is unchanged. --- .../cutlass/qwp/client/QwpWebSocketSender.java | 10 +++++----- .../client/sf/cursor/QwpSchemaCoordinator.java | 15 +++++++++++++++ .../cutlass/qwp/protocol/QwpTableBuffer.java | 12 ++++++++++++ .../cutlass/qwp/protocol/QwpTableBufferTest.java | 14 ++++++++++++++ 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 35219b687..6b16b034f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -4494,23 +4494,23 @@ private QwpSchemaResponse lookUpSchemaForCurrentTable() { // STRICT writes no rows to a table it cannot describe, so no ACK feedback // ever reports that the table shrank: ask again instead of trusting a // cached TOO_LARGE. AUTO keeps the cache hit; its legacy rows earn feedback. - QwpSchemaResponse cached = cursorSendLoop.peekSchema(currentTableName); + QwpSchemaResponse cached = cursorSendLoop.peekSchema(currentTableBuffer.getSchemaKey()); if (cached != null && cached.getResult() == QwpSchemaProtocol.RESULT_TOO_LARGE) { - return cursorSendLoop.refreshSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); + return cursorSendLoop.refreshSchema(currentTableBuffer.getSchemaKey(), remainingSchemaMillis(startNanos, budgetNanos)); } - return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); + return cursorSendLoop.resolveSchema(currentTableBuffer.getSchemaKey(), remainingSchemaMillis(startNanos, budgetNanos)); } if (!cursorSendLoop.hasEverConnected()) { return null; } if (!cursorSendLoop.isWireUp()) { - return cursorSendLoop.peekSchema(currentTableName); + return cursorSendLoop.peekSchema(currentTableBuffer.getSchemaKey()); } if (!cursorSendLoop.isSchemaEnabled()) { return null; } try { - return cursorSendLoop.resolveSchema(currentTableName, remainingSchemaMillis(startNanos, budgetNanos)); + return cursorSendLoop.resolveSchema(currentTableBuffer.getSchemaKey(), remainingSchemaMillis(startNanos, budgetNanos)); } catch (LineSenderSchemaException e) { if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE || e.getReason() == LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java index db35c4d87..7829da2d4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -266,7 +266,22 @@ private static LineSenderSchemaException failure(LineSenderSchemaException.Reaso + ", table=" + table + ", detail=" + detail + ']'); } + /** + * Lower-cases a table name into a cache key. A String that is already lower + * case is its own key, so the sender's pre-computed + * {@code QwpTableBuffer.getSchemaKey()} costs no allocation on a cache hit. + */ private static String normalize(CharSequence tableName) { + if (tableName instanceof String) { + String name = (String) tableName; + for (int i = 0, n = name.length(); i < n; i++) { + char c = name.charAt(i); + if (Character.toLowerCase(c) != c) { + return Chars.toLowerCase(name); + } + } + return name; + } return Chars.toLowerCase(tableName); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 51fee5afb..c20c54bd3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -64,6 +64,8 @@ public class QwpTableBuffer implements QuietCloseable { private static final int MAX_COLUMN_NAME_LENGTH = 127; private final LowerCaseCharSequenceIntHashMap columnNameToIndex; private final ObjList columns; + // lower-cased table name, the schema cache key; the name itself when already lower case + private final String schemaKey; private final QwpWebSocketSender sender; private final String tableName; private QwpColumnDef[] cachedColumnDefs; @@ -87,6 +89,8 @@ public QwpTableBuffer(String tableName) { */ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.tableName = tableName; + String lowerCaseName = Chars.toLowerCase(tableName); + this.schemaKey = tableName.equals(lowerCaseName) ? tableName : lowerCaseName; this.sender = sender; this.columns = new ObjList<>(); this.columnNameToIndex = new LowerCaseCharSequenceIntHashMap(); @@ -295,6 +299,14 @@ public QwpSchemaBinding getSchemaBinding() { return schemaBinding; } + /** + * Returns the lower-cased table name that keys the schema cache, computed + * once so a schema lookup does not allocate. + */ + public String getSchemaKey() { + return schemaKey; + } + void attachSchemaBinding(QwpSchemaBinding binding) { if (schemaBinding != null || rowCount != 0 || hasInProgressRow() || columns.size() != 0) { throw new IllegalStateException("schema binding requires an empty, unbound table buffer"); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index ab14c18d8..1be17770e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1150,6 +1150,20 @@ public void testGetOrCreateColumnThrowsWhenExceedingMaxColumnCount() throws Exce }); } + @Test + public void testGetSchemaKeyLowerCasesOnlyWhenNeeded() throws Exception { + assertMemoryLeak(() -> { + String lowerCaseName = "trades"; + try (QwpTableBuffer table = new QwpTableBuffer(lowerCaseName)) { + assertSame(lowerCaseName, table.getSchemaKey()); + } + try (QwpTableBuffer table = new QwpTableBuffer("Trades_ÄB")) { + assertEquals("trades_äb", table.getSchemaKey()); + assertEquals("Trades_ÄB", table.getTableName()); + } + }); + } + @Test public void testLongArrayMultipleRows() throws Exception { assertMemoryLeak(() -> { From 31219031058f4304a775b87e8c879c6a6db95126 Mon Sep 17 00:00:00 2001 From: Jaromir Hamala Date: Fri, 25 Sep 2026 16:28:36 +0200 Subject: [PATCH 51/51] Tidy schema flag order and stale FdBig comment Move FLAG_SCHEMA into its alphabetical slot after FLAG_GORILLA in QwpConstants. Reword the Unsafe comment on the override field offset: it named the FdBig double-formatting bridge, which this branch deleted, so it now names the makeAccessible() callers that the offset protects. No behavior change. --- .../io/questdb/client/cutlass/qwp/protocol/QwpConstants.java | 4 ++-- core/src/main/java/io/questdb/client/std/Unsafe.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index aac4289c8..77f371301 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -41,8 +41,6 @@ public final class QwpConstants { * but skips the commit until a subsequent message without this flag. */ public static final byte FLAG_DEFER_COMMIT = 0x01; - /** Flag bit: table headers carry schema identity information. */ - public static final byte FLAG_SCHEMA = 0x40; /** * Flag bit: Delta symbol dictionary encoding enabled. * When set, symbol columns use global IDs and send only new dictionary entries. @@ -52,6 +50,8 @@ public final class QwpConstants { * Flag bit: Gorilla timestamp encoding enabled. */ public static final byte FLAG_GORILLA = 0x04; + /** Flag bit: table headers carry schema identity information. */ + public static final byte FLAG_SCHEMA = 0x40; /** * Flag bit: payload region after the prelude is zstd-compressed. Set only * when the handshake negotiated zstd compression. Mirror of the server-side diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java index 1382a1d90..9b115be1a 100644 --- a/core/src/main/java/io/questdb/client/std/Unsafe.java +++ b/core/src/main/java/io/questdb/client/std/Unsafe.java @@ -210,8 +210,8 @@ private static long AccessibleObject_override_fieldOffset() { // object headers (JEP 450, enabled by default in JDK 27) shrink the // header from 12 to 8 bytes, and a hard-coded 12/16 would then point // inside the header -- the Unsafe write to `override` would silently - // miss and setAccessible() would have no effect (surfacing as - // IllegalAccessError from the FdBig double-formatting bridge on JDK 27). + // miss and setAccessible() would have no effect, so makeAccessible() callers + // would fail with IllegalAccessError on JDK 27. // The probe tracks compact (8), compressed (12), uncompressed (16) and // 32-bit (8) layouts automatically. return firstFieldBoundaryOffset();