diff --git a/README.md b/README.md index e945c7770..c461a547a 100644 --- a/README.md +++ b/README.md @@ -555,7 +555,9 @@ Applied by the query pool to select and fail over between the nodes in the `addr | `client_id` | | Opaque client identifier surfaced server-side for observability | The ingest side also accepts store-and-forward and reconnection tuning keys (`auto_flush_*`, `initial_connect_retry`, -`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …). +`reconnect_*`, `request_durable_ack`, `sf_*`, `max_frame_rejections`, `poison_min_escalation_window_millis`, …) and +the schema-aware keys `schema_mode` (`auto`, `strict`, `off`) and `schema_wait_millis`; see +[docs/schema-aware-ingestion.md](docs/schema-aware-ingestion.md). `sf_durability=periodic` checkpoints mmap-published data in the background; `sf_sync_interval_millis` defaults to `5000` in that mode. The interval is a target cadence: JVM scheduling and storage-sync latency add to the actual loss window. Use `request_durable_ack=on` when end-to-end server durability is also required. See the diff --git a/core/pom.xml b/core/pom.xml index 9996932a2..7ad64a627 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -194,7 +194,7 @@ - @@ -220,7 +220,7 @@ + so the Compat shim compiles on each JDK. --> org.codehaus.mojo build-helper-maven-plugin @@ -440,8 +440,8 @@ + Java 9+ Compat shim and module-info.class into the jar ROOT, which + breaks Java 8 consumers. Refuse to publish from any other JDK. --> org.apache.maven.plugins maven-enforcer-plugin @@ -561,13 +561,13 @@ - + org.apache.maven.plugins maven-antrun-plugin diff --git a/core/src/main/java/io/questdb/client/LineSenderSchemaException.java b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java new file mode 100644 index 000000000..3fe7fa5de --- /dev/null +++ b/core/src/main/java/io/questdb/client/LineSenderSchemaException.java @@ -0,0 +1,47 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** A schema-bound row construction failure with a stable reason. */ +public class LineSenderSchemaException extends LineSenderException { + public enum Reason { + INVALID_VALUE, + SCHEMA_UNAVAILABLE, + ACCESS_DENIED, + UNSUPPORTED_FEATURE + } + + private final Reason reason; + + public LineSenderSchemaException(Reason reason, CharSequence message) { + super(message, false); + this.reason = reason; + } + + public Reason getReason() { + return reason; + } +} diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 645d7b254..284706727 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -502,7 +502,8 @@ default Sender decimalColumn(CharSequence name, CharSequence value) { * * @param timeoutMillis upper bound on the wait; {@code <= 0} returns the * current state without blocking (the flush still - * happens before the check) + * happens before the check), and {@code Long.MAX_VALUE} + * waits without a limit * @return {@code true} if the server has acknowledged every published * frame on return, {@code false} on timeout * @throws LineSenderException if the transport has latched a terminal error @@ -570,6 +571,10 @@ default long flushAndGetSequence() { * Precision is locked the first time a value is added to the column: subsequent * rows must use the same precision or a {@link LineSenderException} is thrown. * To mark the value NULL, do not call this method for the current row. + * In schema-aware mode, the server column must be a GEOHASH with exactly the + * same precision, STRING, or VARCHAR. Text targets receive a binary string + * containing exactly {@code precisionBits} characters. Legacy mode keeps the + * native GEOHASH wire representation and server-side conversion behavior. * * @param name name of the column * @param bits packed geohash; low {@code precisionBits} bits significant @@ -595,6 +600,10 @@ default Sender geoHashColumn(CharSequence name, long bits, int precisionBits) { *

* The first call locks the column at {@code value.length() * 5} bits; all * subsequent rows must supply strings of the same length. + * In schema-aware mode, the server column must be a GEOHASH with exactly that + * precision, STRING, or VARCHAR. Text targets receive the corresponding binary + * bit string. Legacy mode keeps the native GEOHASH wire representation and + * server-side conversion behavior. * * @param name name of the column * @param value base32 geohash string, 1..12 characters; must not be null or empty @@ -653,10 +662,18 @@ default Sender ipv4Column(CharSequence name, int address) { /** * Add an IPv4 column value from a dotted-quad string (e.g. "192.168.1.1"). * + * A null reference skips the column for the current row. The literal + * {@code "null"} (case-insensitive) and the exact address + * {@code "0.0.0.0"} are rejected; omit the setter or pass a null reference + * to write a missing value. + * Legacy dotted aliases such as {@code ".0.0.0.0."} are accepted and + * normalized to the IPv4 NULL value. + * * @param name name of the column - * @param address dotted-quad IPv4 address; must not be null + * @param address dotted-quad IPv4 address, or null to skip the column * @return this instance for method chaining - * @throws LineSenderException if the address fails to parse, or the + * @throws LineSenderException if the address fails to parse, represents a + * rejected NULL sentinel, or the * configured protocol version does not support IPv4 */ default Sender ipv4Column(CharSequence name, CharSequence address) { @@ -666,6 +683,10 @@ default Sender ipv4Column(CharSequence name, CharSequence address) { /** * Add a LONG256 column value, packed as four 64-bit words, least-significant first * (so the 256-bit value is {@code (l3 << 192) | (l2 << 128) | (l1 << 64) | l0}). + * In schema-aware mode, setting all four words to {@link Long#MIN_VALUE} + * writes SQL NULL. Any other combination, including a partial match, is a value. + * Legacy mode preserves its existing behavior: the four words are emitted as + * native LONG256 wire data and server-side null/conversion handling applies. * * @param name name of the column * @param l0 bits 0..63 (least significant) @@ -745,6 +766,13 @@ default Sender shortColumn(CharSequence name, short value) { * to finalize the row. You can then start a new row by calling this method again. *
* If you want to cancel the current row, you can call {@link #cancelRow()}. + * QWP WebSocket senders resolve the table's schema here when the server speaks schema-aware QWP. The call may + * therefore wait up to {@code schema_wait_millis} (with an asynchronous initial connect, that includes waiting for + * the first connection), and it publishes the pending batch when this table's wire contract differs from it. So + * schema errors, and the flush-timeout and backpressure errors of that publish, can surface here, before the row + * has any values. A failure leaves the table selected and no row started; calling this method again retries the + * resolution. {@link SchemaMode} decides what happens when the schema is not obtainable. Setters may resolve it + * again when an implicitly started row crosses a batch boundary. * * @param table name of the table * @return this instance for method chaining @@ -799,7 +827,10 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Transport failures (unreachable or + * engine until the wire is up. {@link SchemaMode} decides whether a + * row written before the first upgrade waits for schema negotiation + * or uses the legacy conversion contract. Transport failures + * (unreachable or * dropped server) retry indefinitely and are never surfaced -- the * buffered rows are safe in SF and the server may still appear. A * terminal auth, upgrade or capability rejection on the initial @@ -824,6 +855,52 @@ enum InitialConnectMode { ASYNC } + /** + * Row-encoding contract of the WebSocket (QWP) sender when the server speaks + * schema-aware QWP. It decides what a row does when its table's schema is + * not obtainable at the moment the row starts: before the first upgrade + * completes under {@link InitialConnectMode#ASYNC}, while the wire is down + * during an outage, or while a schema lookup is still pending. It is + * independent of {@link InitialConnectMode}, which only governs how + * {@code fromConfig} behaves before the first successful upgrade. + *

    + *
  • {@link #AUTO} (default) — a row uses the strict schema contract + * whenever a schema is obtainable: from the batch's pinned snapshot, + * from the cache, or from a lookup that completes within + * {@code schema_wait_millis} while the wire is up. Otherwise the row + * uses the legacy conversion contract; schema unavailability never + * fails a row and the producer never waits on a wire that is down. + * Rows written that way keep store-and-forward's promise that an + * outage never blocks the producer, but the server, not the client, + * validates them, and values such as nanosecond timestamps use their + * legacy representation.
    • + *
  • {@link #STRICT} — once the server is known to speak schema-aware + * QWP, no row is ever encoded with the legacy contract. A row whose + * schema is not obtainable waits up to {@code schema_wait_millis}, + * across reconnects, and then fails with + * {@link LineSenderSchemaException} ({@code SCHEMA_UNAVAILABLE}). + * Under {@link InitialConnectMode#ASYNC} the first row also waits for + * the first upgrade, because the server's capability is unknown until + * then.
    • + *
  • {@link #OFF} — every row uses the legacy conversion contract and + * the sender never looks up schemas. It requests schema support only + * when its stream must replay schema-encoded backlog recovered from + * a store-and-forward directory. Legacy-only streams can fail over + * to servers without schema support.
    • + *
+ *

+ * A batch keeps the contract under which its first row was written: rows + * added to a pending legacy batch stay legacy even after a schema becomes + * obtainable, and a pending schema batch is published before a legacy row + * starts a new batch. STRICT permits legacy rows only until a connection + * successfully negotiates schema support. + */ + enum SchemaMode { + AUTO, + STRICT, + OFF + } + /** * Durability contract for the store-and-forward write path. Selects when * the SF segment file is fsynced; trades latency / throughput for @@ -1147,6 +1224,14 @@ public int getConnectTimeout() { private long reconnectMaxDurationMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean requestDurableAck; private int retryTimeoutMillis = PARAMETER_NOT_SET_EXPLICITLY; + // Row-encoding contract when a schema is not obtainable (WebSocket + // only). null means "not set explicitly", which build() resolves to + // AUTO. + private SchemaMode schemaMode = null; + // Upper bound for one schema wait: the initial handshake, if still + // pending, plus one DESCRIBE round trip. Reconnects consume the same + // budget. + private long schemaWaitMillis = PARAMETER_NOT_SET_EXPLICITLY; private boolean transactional; private String senderId = DEFAULT_SENDER_ID; // Per-append deadline for SF appendBlocking spin-then-throw. Used to @@ -1533,6 +1618,10 @@ public Sender build() { } else { actualInitialConnectMode = InitialConnectMode.OFF; } + SchemaMode actualSchemaMode = schemaMode == null ? SchemaMode.AUTO : schemaMode; + long actualSchemaWaitMillis = schemaWaitMillis == PARAMETER_NOT_SET_EXPLICITLY + ? QwpWebSocketSender.DEFAULT_SCHEMA_WAIT_MILLIS + : schemaWaitMillis; long actualDurableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis == DURABLE_ACK_KEEPALIVE_NOT_SET ? CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS @@ -1714,7 +1803,9 @@ public Sender build() { actualConnectionListenerInboxCapacity, actualMaxFrameRejections, actualPoisonMinEscalationWindowMillis, - actualCatchUpCapGapMinEscalationWindowMillis + actualCatchUpCapGapMinEscalationWindowMillis, + actualSchemaMode, + actualSchemaWaitMillis ); } catch (UnreplayableSlotException e) { // The one failure build() recovers from. The slot's frames reference ids @@ -1914,7 +2005,7 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { * {@link Sender#drain(long)} explicitly before close(). *

* Set to {@code 0} or {@code -1} to opt out — close() will not wait - * at all (fast close). Pending data is then lost in memory mode and + * at all (fast close). {@code Long.MAX_VALUE} waits without a limit. Pending data is then lost in memory mode and * recovered by the next sender in SF mode. *

* WebSocket transport only. @@ -2941,6 +3032,40 @@ public boolean isStoreAndForwardEnabled() { return sfDir != null; } + /** + * Row-encoding contract when a table's schema is not obtainable -- + * see {@link SchemaMode} for the value semantics. Default + * {@link SchemaMode#AUTO}. WebSocket transport only. + */ + public LineSenderBuilder schemaMode(SchemaMode mode) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_mode is only supported for WebSocket transport"); + } + if (mode == null) { + throw new LineSenderException("schema_mode cannot be null"); + } + this.schemaMode = mode; + return this; + } + + /** + * Upper bound for one schema wait: the first WebSocket upgrade, if it + * has not completed yet, plus one schema lookup round trip. Reconnects + * consume the same budget; they do not restart it. Default 30 s; + * {@code Long.MAX_VALUE} waits without a limit. At the bound, {@link SchemaMode#AUTO} writes the row with the legacy contract + * and {@link SchemaMode#STRICT} fails it. WebSocket transport only. + */ + public LineSenderBuilder schemaWaitMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_wait_millis is only supported for WebSocket transport"); + } + if (millis <= 0) { + throw new LineSenderException("schema_wait_millis must be > 0: ").put(millis); + } + this.schemaWaitMillis = millis; + return this; + } + /** * Per-call deadline for {@code Sender.flush()} spinning on a full * cursor segment ring waiting for ACKs to drain space. Default @@ -3881,6 +4006,18 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } else { throw new LineSenderException("invalid initial_connect_retry [value=").put(sink).put(", allowed-values=[on, off, true, false, sync, async]]"); } + } else if (Chars.equals("schema_mode", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_mode is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "schema_mode"); + schemaMode(parseSchemaModeValue(sink)); + } else if (Chars.equals("schema_wait_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("schema_wait_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "schema_wait_millis"); + schemaWaitMillis(parseLongValue(sink, "schema_wait_millis")); } else if (Chars.equals("sf_append_deadline_millis", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("sf_append_deadline_millis is only supported for WebSocket transport"); @@ -4144,6 +4281,15 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } + if (view.has("schema_wait_millis")) { + schemaWaitMillis(wsLong(view, v, "schema_wait_millis")); + } + s = view.getStr("schema_mode"); + if (s != null) { + v.clear(); + v.put(s); + schemaMode(parseSchemaModeValue(v)); + } if (view.has("sf_max_segment_bytes")) { storeAndForwardMaxSegmentBytes(wsSize(view, v, "sf_max_segment_bytes")); } @@ -4281,6 +4427,20 @@ private static int wsInt(ConfigView view, StringSink v, String key) { return parseIntValue(v, key); } + private static SchemaMode parseSchemaModeValue(CharSequence value) { + if (Chars.equalsIgnoreCase("auto", value)) { + return SchemaMode.AUTO; + } + if (Chars.equalsIgnoreCase("strict", value)) { + return SchemaMode.STRICT; + } + if (Chars.equalsIgnoreCase("off", value)) { + return SchemaMode.OFF; + } + throw new LineSenderException("invalid schema_mode [value=").put(value) + .put(", allowed-values=[auto, strict, off]]"); + } + private static long wsLong(ConfigView view, StringSink v, String key) { v.clear(); v.put(view.getStr(key)); @@ -4313,6 +4473,8 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("sf_max_total_bytes", sfMaxTotalBytes); m.put("sf_durability", sfDurability == null ? null : sfDurability.name()); m.put("sf_append_deadline_millis", sfAppendDeadlineMillis); + m.put("schema_mode", schemaMode == null ? null : schemaMode.name()); + m.put("schema_wait_millis", schemaWaitMillis); m.put("sf_sync_interval_millis", sfSyncIntervalMillis); m.put("close_flush_timeout_millis", closeFlushTimeoutMillis); m.put("durable_ack_keepalive_interval_millis", durableAckKeepaliveIntervalMillis); diff --git a/core/src/main/java/io/questdb/client/cairo/ColumnType.java b/core/src/main/java/io/questdb/client/cairo/ColumnType.java index f87d6f322..1e4e4816b 100644 --- a/core/src/main/java/io/questdb/client/cairo/ColumnType.java +++ b/core/src/main/java/io/questdb/client/cairo/ColumnType.java @@ -231,12 +231,24 @@ public static int getDecimalType(int precision, int scale) { return ((scale & 0xFF) << 18) | ((precision & 0xFF) << 8) | (DECIMAL8 + size); } + public static int getDecimalPrecision(int type) { + return (type >>> 8) & 0xFF; + } + + public static int getDecimalScale(int type) { + return (type >>> 18) & 0xFF; + } + public static int getGeoHashTypeWithBits(int bits) { assert bits > 0 && bits <= GEOLONG_MAX_BITS; // this logic relies on GeoHash type value to be clustered together return mkGeoHashType(bits, (short) (GEOBYTE + pow2SizeOfBits(bits))); } + public static int getGeoHashBits(int type) { + return (type >>> BYTE_BITS) & 0xff; + } + /** * Is an N-dimensional array type. */ @@ -244,6 +256,15 @@ public static boolean isArray(int columnType) { return ColumnType.tagOf(columnType) == ColumnType.ARRAY; } + public static boolean isDecimal(int columnType) { + short tag = tagOf(columnType); + return tag >= DECIMAL8 && tag <= DECIMAL256; + } + + public static boolean isGeoHash(int columnType) { + return (columnType & TYPE_FLAG_GEO_HASH) != 0; + } + public static boolean isNull(int columnType) { return columnType == NULL; } diff --git a/core/src/main/java/io/questdb/client/cairo/TableUtils.java b/core/src/main/java/io/questdb/client/cairo/TableUtils.java index f250e9f80..6b13bea3b 100644 --- a/core/src/main/java/io/questdb/client/cairo/TableUtils.java +++ b/core/src/main/java/io/questdb/client/cairo/TableUtils.java @@ -87,6 +87,15 @@ public static boolean isValidTableName(CharSequence tableName, int fsFileNameLim for (int i = 0; i < length; i++) { char c = tableName.charAt(i); + if (Character.isHighSurrogate(c)) { + if (++i >= length || !Character.isLowSurrogate(tableName.charAt(i))) { + return false; + } + continue; + } + if (Character.isLowSurrogate(c)) { + return false; + } switch (c) { case '.': if (i == 0 || i == length - 1 || tableName.charAt(i - 1) == '.') { diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index 1290fcab5..333f92f57 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -83,6 +83,8 @@ public abstract class WebSocketClient implements QuietCloseable { private static final String QWP_DURABLE_ACK_ENABLED_VALUE = "enabled"; private static final String QWP_DURABLE_ACK_HEADER_NAME = "X-QWP-Durable-Ack:"; private static final String QWP_MAX_BATCH_SIZE_HEADER_NAME = "X-QWP-Max-Batch-Size:"; + private static final String QWP_SCHEMA_ENABLED_VALUE = "enabled"; + private static final String QWP_SCHEMA_HEADER_NAME = "X-QWP-Schema:"; private static final String QWP_VERSION_HEADER_NAME = "X-QWP-Version:"; private static final ThreadLocal SHA1_DIGEST = ThreadLocal.withInitial(() -> { try { @@ -138,6 +140,9 @@ public abstract class WebSocketClient implements QuietCloseable { private int qwpMaxVersion = 1; // Opt-in for STATUS_DURABLE_ACK frames; sent as X-QWP-Request-Durable-Ack: true private boolean qwpRequestDurableAck; + // Schema negotiation is deliberately opt-in at this low-level client layer. + // The public Sender does not activate it until schema-directed encoding is available. + private boolean qwpRequestSchema; // Receive buffer (native memory) private long recvBufPtr; private int recvBufSize; @@ -149,6 +154,7 @@ public abstract class WebSocketClient implements QuietCloseable { // path can rely on durable-ack-driven trim. Absence (after opting in via // setQwpRequestDurableAck) is the early-fail signal. private boolean serverDurableAckEnabled; + private boolean serverQwpSchemaEnabled; // Server's hard cap on ingest QWP message payload bytes, extracted from // X-QWP-Max-Batch-Size on the 101 upgrade response. 0 when the server did // not advertise the header (older builds), in which case the sender falls @@ -291,6 +297,7 @@ public void connect(CharSequence host, int port) { public void disconnect() { Misc.free(socket); upgraded = false; + serverQwpSchemaEnabled = false; host = null; port = 0; recvPos = 0; @@ -410,6 +417,11 @@ public boolean isServerDurableAckEnabled() { return serverDurableAckEnabled; } + /** Returns whether a requested schema extension was confirmed by this connection. */ + public boolean isQwpSchemaEnabled() { + return serverQwpSchemaEnabled; + } + /** * Receives and processes WebSocket frames. * @@ -469,6 +481,19 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** Sends a small caller-owned byte array as one binary message. */ + public void sendBinary(byte[] data, int timeout) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + for (byte b : data) { + sendBuffer.putByte(b); + } + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends two native-memory slices as one WebSocket binary frame. The slices * are copied directly into the WebSocket send buffer and masked there, so a @@ -595,6 +620,14 @@ public void setQwpRequestDurableAck(boolean enabled) { this.qwpRequestDurableAck = enabled; } + /** + * Requests schema control support on subsequent WebSocket upgrades. + * This operation is intentionally one-way; there is no low-level disable method. + */ + public void requestQwpSchema() { + this.qwpRequestSchema = true; + } + /** * Non-blocking attempt to receive a WebSocket frame. * Returns immediately if no complete frame is available. @@ -650,6 +683,7 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe upgradeRejectRole = null; upgradeRejectZone = null; upgradeStatusCode = 0; + serverQwpSchemaEnabled = false; // Generate random key byte[] keyBytes = new byte[16]; @@ -697,6 +731,9 @@ public void upgrade(CharSequence path, int timeout, CharSequence authorizationHe if (qwpRequestDurableAck) { sendBuffer.putAscii("X-QWP-Request-Durable-Ack: true\r\n"); } + if (qwpRequestSchema) { + sendBuffer.putAscii("X-QWP-Request-Schema: true\r\n"); + } if (authorizationHeader != null) { sendBuffer.putAscii("Authorization: "); sendBuffer.putAscii(authorizationHeader); @@ -753,6 +790,28 @@ private static boolean excludesHeaderValue(String response, String headerName, S return true; } + private static String extractUniqueHeaderValue(String response, String headerName) { + String value = null; + int lineStart = response.indexOf("\r\n") + 2; // skip status line + while (lineStart >= 2 && lineStart < response.length()) { + int lineEnd = response.indexOf("\r\n", lineStart); + if (lineEnd < 0 || lineEnd == lineStart) { + break; + } + int colon = response.indexOf(':', lineStart); + if (colon > lineStart && colon < lineEnd + && colon - lineStart == headerName.length() - 1 + && response.regionMatches(true, lineStart, headerName, 0, headerName.length() - 1)) { + if (value != null) { + throw new HttpClientException("Duplicate X-QWP-Schema header"); + } + value = response.substring(colon + 1, lineEnd).trim(); + } + lineStart = lineEnd + 2; + } + return value; + } + /** * Extracts the zstd level the server applied from the echoed * {@code X-QWP-Content-Encoding} response header. Returns 0 when the @@ -1392,6 +1451,17 @@ private void validateUpgradeResponse(int headerEnd) { // missing trim signals. serverDurableAckEnabled = extractDurableAckEnabled(response); + // An absent confirmation means an older server. A present value is a + // protocol assertion, so a requested handshake must reject anything + // other than the single defined value instead of treating it as legacy. + if (qwpRequestSchema) { + String schemaValue = extractUniqueHeaderValue(response, QWP_SCHEMA_HEADER_NAME); + if (schemaValue != null && !schemaValue.equalsIgnoreCase(QWP_SCHEMA_ENABLED_VALUE)) { + throw new HttpClientException("Invalid X-QWP-Schema header value: ").put(schemaValue); + } + serverQwpSchemaEnabled = schemaValue != null; + } + // Extract X-QWP-Max-Batch-Size (optional). Older servers omit it; the // sender falls back to its locally configured byte budget in that case. serverMaxBatchSize = extractMaxBatchSize(response); diff --git a/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java b/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java index 10c56d5a8..026543481 100644 --- a/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java +++ b/core/src/main/java/io/questdb/client/cutlass/line/array/AbstractArray.java @@ -147,6 +147,19 @@ public void clear() { memA = array.startMemoryA(); } + /** + * Returns the number of dimensions in this array. + * + * @return the array dimensionality + * @throws LineSenderException if the array is closed + */ + public int getDimensionality() { + if (closed) { + throw new LineSenderException("Cannot inspect a closed array"); + } + return array.getDimCount(); + } + /** * Closes this array and releases all associated native memory resources. *

@@ -326,4 +339,4 @@ protected int toFlatOffset(int[] coords) { } return flatOffset; } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java index f8cc669fb..11ec5ce13 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java @@ -53,11 +53,12 @@ private void encodeColumn( long stringDataSize, int symbolDictionarySize, boolean useGorilla, - boolean useGlobalSymbols + boolean useGlobalSymbols, + boolean forceNullBitmap ) { long dataAddr = col.getDataAddress(); - writeNullHeader(col, rowCount, rowCount - valueCount); + writeNullHeader(col, rowCount, rowCount - valueCount, forceNullBitmap); switch (col.getType()) { case TYPE_BOOLEAN: @@ -233,8 +234,13 @@ private void writeLongArrayColumn(QwpTableBuffer.ColumnBuffer col, int count) { } } - private void writeNullHeader(QwpTableBuffer.ColumnBuffer col, int rowCount, int nullCount) { - if (nullCount > 0) { + private void writeNullHeader( + QwpTableBuffer.ColumnBuffer col, + int rowCount, + int nullCount, + boolean forceNullBitmap + ) { + if (nullCount > 0 || forceNullBitmap) { buffer.putByte((byte) 1); col.ensureNullBitmapCapacity(rowCount); long nullAddr = col.getNullBitmapAddress(); @@ -281,6 +287,24 @@ private void writeSymbolColumnWithGlobalIds(QwpTableBuffer.ColumnBuffer col, int private void writeTableHeader(String tableName, int rowCount, QwpColumnDef[] columns) { buffer.putString(tableName); + writeTableShape(rowCount, columns); + } + + private void writeSchemaTableHeader(String tableName, int tableId, long metadataVersion, int rowCount, QwpColumnDef[] columns) { + buffer.putString(tableName); + if (tableId == -1 && metadataVersion == -1) { + buffer.putByte((byte) 0); + } else if (tableId >= 0 && metadataVersion >= 0) { + buffer.putByte((byte) 1); + buffer.putInt(tableId); + buffer.putLong(metadataVersion); + } else { + throw new IllegalArgumentException("schema identity must be both known or both unknown"); + } + writeTableShape(rowCount, columns); + } + + private void writeTableShape(int rowCount, QwpColumnDef[] columns) { buffer.putVarint(rowCount); buffer.putVarint(columns.length); for (QwpColumnDef col : columns) { @@ -319,6 +343,17 @@ void encodeTable(QwpTableBuffer tableBuffer, boolean useGlobalSymbols, boolean u encodeTable(tableBuffer, tableBuffer.getRowCount(), null, null, null, useGlobalSymbols, useGorilla); } + void encodeSchemaTable(QwpTableBuffer tableBuffer, int tableId, long metadataVersion, boolean useGlobalSymbols, boolean useGorilla) { + QwpColumnDef[] columnDefs = tableBuffer.getColumnDefs(); + writeSchemaTableHeader(tableBuffer.getTableName(), tableId, metadataVersion, tableBuffer.getRowCount(), columnDefs); + for (int i = 0; i < tableBuffer.getColumnCount(); i++) { + QwpTableBuffer.ColumnBuffer col = tableBuffer.getColumn(i); + encodeColumn(col, tableBuffer.getRowCount(), col.getValueCount(), col.getStringDataSize(), + col.getSymbolDictionarySize(), useGorilla, useGlobalSymbols, + col.getType() == TYPE_GEOHASH && col.usesNullBitmap()); + } + } + void encodeTable( QwpTableBuffer tableBuffer, int rowCount, @@ -344,7 +379,8 @@ void encodeTable( symbolDictionarySize = limitedSymbolDictionarySizes[i]; } - encodeColumn(col, rowCount, valueCount, stringDataSize, symbolDictionarySize, useGorilla, useGlobalSymbols); + encodeColumn(col, rowCount, valueCount, stringDataSize, symbolDictionarySize, + useGorilla, useGlobalSymbols, false); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java new file mode 100644 index 000000000..122beb47d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpSchemaCapabilityMismatchException.java @@ -0,0 +1,29 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client; + +import io.questdb.client.cutlass.http.client.HttpClientException; + +/** A connection cannot replay queued schema-framed data without schema support. */ +public final class QwpSchemaCapabilityMismatchException extends HttpClientException { + public QwpSchemaCapabilityMismatchException() { + super("WebSocket upgrade succeeded but server does not support schema-framed QWP backlog"); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index f38d9e93d..ad8cbfd11 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; @@ -56,6 +57,7 @@ public class QwpWebSocketEncoder implements QuietCloseable { // values when delta-of-delta overflows int32. private byte flags = FLAG_GORILLA; private int payloadStart; + private boolean schemaMessage; private byte version = VERSION; public QwpWebSocketEncoder() { @@ -67,23 +69,57 @@ public QwpWebSocketEncoder(int bufferSize) { } public void addTable(QwpTableBuffer tableBuffer) { + rejectBoundLegacyTable(tableBuffer); + if (schemaMessage) { + throw new IllegalStateException("legacy table cannot be added to a schema message"); + } columnWriter.encodeTable(tableBuffer, true, true); } + public void addSchemaTable(QwpTableBuffer tableBuffer, int tableId, long metadataVersion) { + validateBoundSchemaTable(tableBuffer, tableId, metadataVersion, true); + if (!schemaMessage) { + throw new IllegalStateException("schema table requires a schema message"); + } + validateIdentity(tableId, metadataVersion); + columnWriter.encodeSchemaTable(tableBuffer, tableId, metadataVersion, true, true); + } + public void beginMessage( int tableCount, GlobalSymbolDictionary globalDict, int confirmedMaxId, int batchMaxId + ) { + schemaMessage = false; + beginMessage0(tableCount, globalDict, confirmedMaxId, batchMaxId); + } + + public void beginSchemaMessage( + int tableCount, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId + ) { + if (tableCount <= 0) { + throw new IllegalArgumentException("schema message must contain at least one table"); + } + rejectControlFlag(); + schemaMessage = true; + beginMessage0(tableCount, globalDict, confirmedMaxId, batchMaxId); + } + + private void beginMessage0( + int tableCount, + GlobalSymbolDictionary globalDict, + int confirmedMaxId, + int batchMaxId ) { buffer.reset(); deltaStart = confirmedMaxId + 1; deltaCount = Math.max(0, batchMaxId - confirmedMaxId); - byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); - byte origFlags = flags; - flags = headerFlags; - writeHeader(tableCount, 0); - flags = origFlags; + byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT | (schemaMessage ? FLAG_SCHEMA : 0)); + writeHeader(tableCount, 0, headerFlags); payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); @@ -163,6 +199,8 @@ public int copySplitMessage( } public int encode(QwpTableBuffer tableBuffer) { + rejectBoundLegacyTable(tableBuffer); + schemaMessage = false; buffer.reset(); writeHeader(1, 0); int payloadStart = buffer.getPosition(); @@ -173,12 +211,38 @@ public int encode(QwpTableBuffer tableBuffer) { return buffer.getPosition(); } + public int encodeSchema(QwpTableBuffer tableBuffer, int tableId, long metadataVersion) { + validateIdentity(tableId, metadataVersion); + validateBoundSchemaTable(tableBuffer, tableId, metadataVersion, false); + if (tableBuffer.getSchemaBinding() != null && tableBuffer.getRowCount() == 0) { + return 0; + } + rejectControlFlag(); + schemaMessage = true; + buffer.reset(); + writeHeader(1, 0, (byte) (flags | FLAG_SCHEMA)); + int payloadStart = buffer.getPosition(); + columnWriter.setBuffer(buffer); + columnWriter.encodeSchemaTable(tableBuffer, tableId, metadataVersion, false, true); + buffer.patchInt(8, buffer.getPosition() - payloadStart); + return buffer.getPosition(); + } + + public int encodeSchema(QwpTableBuffer tableBuffer) { + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if (binding == null) { + throw new IllegalStateException("schema encoding requires an attached schema binding"); + } + return encodeSchema(tableBuffer, binding.getTableId(), binding.getMetadataVersion()); + } + public int encodeWithDeltaDict( QwpTableBuffer tableBuffer, GlobalSymbolDictionary globalDict, int confirmedMaxId, int batchMaxId ) { + rejectBoundLegacyTable(tableBuffer); beginMessage(1, globalDict, confirmedMaxId, batchMaxId); addTable(tableBuffer); return finishMessage(); @@ -190,6 +254,39 @@ public int finishMessage() { return buffer.getPosition(); } + private static void validateIdentity(int tableId, long metadataVersion) { + if (!((tableId == -1 && metadataVersion == -1) || (tableId >= 0 && metadataVersion >= 0))) { + throw new IllegalArgumentException("schema identity must be both known or both unknown"); + } + } + + private static void rejectBoundLegacyTable(QwpTableBuffer tableBuffer) { + if (tableBuffer.getSchemaBinding() != null) { + throw new IllegalStateException("schema-bound table cannot use legacy framing"); + } + } + + private static void validateBoundSchemaTable( + QwpTableBuffer tableBuffer, + int tableId, + long metadataVersion, + boolean additive + ) { + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if (binding == null) { + return; + } + if (binding.getTableId() != tableId || binding.getMetadataVersion() != metadataVersion) { + throw new IllegalArgumentException("explicit schema identity does not match the attached binding"); + } + if (tableBuffer.hasInProgressRow()) { + throw new IllegalStateException("cannot encode a schema-bound table with an incomplete row"); + } + if (additive && tableBuffer.getRowCount() == 0) { + throw new IllegalStateException("cannot add an empty schema-bound table"); + } + } + public QwpBufferWriter getBuffer() { return buffer; } @@ -234,16 +331,29 @@ public void setVersion(byte version) { } public void writeHeader(int tableCount, int payloadLength) { + writeHeader(tableCount, payloadLength, flags); + } + + private void writeHeader(int tableCount, int payloadLength, byte headerFlags) { + if (tableCount < 0 || tableCount > 0xffff) { + throw new IllegalArgumentException("QWP table count is outside unsigned-short range: " + tableCount); + } buffer.putByte((byte) 'Q'); buffer.putByte((byte) 'W'); buffer.putByte((byte) 'P'); buffer.putByte((byte) '1'); buffer.putByte(version); - buffer.putByte(flags); + buffer.putByte(headerFlags); buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + private void rejectControlFlag() { + if ((flags & io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol.FLAG_CONTROL) != 0) { + throw new IllegalStateException("schema data messages cannot use the control flag"); + } + } + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { if (splitDeltaCount == 0) { return 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 41cc0a8c9..6b16b034f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client; import io.questdb.client.ClientTlsConfiguration; +import io.questdb.client.LineSenderSchemaException; import io.questdb.client.Sender; import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderConnectionListener; @@ -48,6 +49,9 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.MmapSegment; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; @@ -145,6 +149,10 @@ public class QwpWebSocketSender implements Sender { // sf-client.md section 4.4 floor: drop-oldest under bursts needs a wide // enough window to preserve the trailing category distribution. private static final int MIN_ERROR_INBOX_CAPACITY = 16; + // Upper bound for a row's first setter to wait on the initial handshake + // and a schema DESCRIBE round trip. A healthy lookup takes milliseconds; + // this only bounds a dead or still-connecting link. + public static final long DEFAULT_SCHEMA_WAIT_MILLIS = 30_000L; private static final String WRITE_PATH = "/write/v4"; // Yields the Authorization header value presented on each WebSocket upgrade. A constant for a // fixed token or Basic credential; for an httpTokenProvider it pulls a freshly refreshed token, @@ -324,6 +332,9 @@ public class QwpWebSocketSender implements Sender { // (Invariant B); endpoint-policy and transport failures stay // contained in that loop and never reach the producer. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; + // Wire contract of the pending batch, selected by its first committed row. + // Meaningful only while pendingRowCount > 0. Producer-thread only. + private boolean isPendingBatchLegacy; private boolean ownsCursorEngine; // Whether close() may let the engine reclaim the parent-anchored LOGICAL slot lock. // False only while an outer frame holds it: Sender.build() acquires it for the whole @@ -385,6 +396,10 @@ public class QwpWebSocketSender implements Sender { private long reconnectMaxDurationMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_MAX_DURATION_MILLIS; private boolean requestDurableAck; + // Row-encoding contract when a schema is not obtainable; see + // resolveSchemaForCurrentTable(). + private Sender.SchemaMode schemaMode = Sender.SchemaMode.AUTO; + private long schemaWaitMillis = DEFAULT_SCHEMA_WAIT_MILLIS; // Monotonic per-attempt counter snapshotted onto every connection event // fired from buildAndConnect. Counts every FOREGROUND endpoint try -- // successes and failures alike -- across this sender's lifetime. @@ -855,11 +870,51 @@ autoFlushIntervalNanos, fixedAuthHeader(authorizationHeader), catchUpCapGapMinEscalationWindowMillis); } + /** + * Overload with the default {@link Sender.SchemaMode#AUTO} schema contract + * and the default schema wait. + */ + public static QwpWebSocketSender connectWithCredentialSupplier( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + Supplier authorizationHeaderSupplier, + boolean requestDurableAck, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs, + int connectTimeoutMs, + SenderConnectionListener connectionListener, + int connectionListenerInboxCapacity, + int maxFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis + ) { + return connectWithCredentialSupplier(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, + autoFlushIntervalNanos, authorizationHeaderSupplier, requestDurableAck, cursorEngine, + closeFlushTimeoutMillis, reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, + connectionListenerInboxCapacity, maxFrameRejections, poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, Sender.SchemaMode.AUTO, DEFAULT_SCHEMA_WAIT_MILLIS); + } + /** * Master connect entry point — also accepts the poison-frame detector * threshold ({@code max_frame_rejections}): consecutive server-active * rejections of the same head-of-line frame, with no ack progress in - * between, before the loop escalates to a typed terminal. + * between, before the loop escalates to a typed terminal; and the schema + * contract ({@code schema_mode}) with its lookup budget + * ({@code schema_wait_millis}). *

* Named apart from {@code connect} for the reason given on * {@link #connectWithCredentialSupplier(List, ClientTlsConfiguration, int, int, long, Supplier, @@ -889,7 +944,9 @@ public static QwpWebSocketSender connectWithCredentialSupplier( int connectionListenerInboxCapacity, int maxFrameRejections, long poisonMinEscalationWindowMillis, - long catchUpCapGapMinEscalationWindowMillis + long catchUpCapGapMinEscalationWindowMillis, + Sender.SchemaMode schemaMode, + long schemaWaitMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -911,6 +968,8 @@ public static QwpWebSocketSender connectWithCredentialSupplier( sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; + sender.schemaMode = schemaMode == null ? Sender.SchemaMode.AUTO : schemaMode; + sender.schemaWaitMillis = schemaWaitMillis; if (errorHandler != null) { sender.setErrorHandler(errorHandler); } @@ -1014,6 +1073,12 @@ public void at(long timestamp, ChronoUnit unit) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.designatedTimestamp(timestamp, unit); + sendRow(); + return; + } if (unit == ChronoUnit.NANOS) { atNanos(timestamp); } else { @@ -1021,8 +1086,7 @@ public void at(long timestamp, ChronoUnit unit) { atMicros(micros); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } } @@ -1031,11 +1095,16 @@ public void at(Instant timestamp) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.designatedTimestamp(timestamp); + sendRow(); + return; + } long micros = timestamp.getEpochSecond() * 1_000_000L + timestamp.getNano() / 1000L; atMicros(micros); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } } @@ -1044,11 +1113,11 @@ public void atNow() { checkNotClosed(); checkTableSelected(); try { + bindingForEffectiveWrite(); // Server-assigned timestamp - just send the row without designated timestamp sendRow(); } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } } @@ -1088,14 +1157,17 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { if (timeoutMillis <= 0L) { return false; } - long deadlineNanos = System.nanoTime() + timeoutMillis * 1_000_000L; + // Elapsed-versus-budget, not an absolute deadline: toNanos saturates, so + // Long.MAX_VALUE waits indefinitely instead of wrapping to no wait. + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); while (cursorEngine.ackedFsn() < targetFsn) { cursorEngine.checkDurability(); if (cursorSendLoop != null) { cursorSendLoop.checkError(); } checkConnectionError(); - if (System.nanoTime() >= deadlineNanos) { + if (System.nanoTime() - startNanos >= budgetNanos) { return false; } java.util.concurrent.locks.LockSupport.parkNanos(50_000L); @@ -1107,26 +1179,29 @@ public boolean awaitAckedFsn(long targetFsn, long timeoutMillis) { * Adds a BINARY column value to the current row. The bytes are written * verbatim with no encoding or transformation. A {@code null} array * reference is rejected so the NULL contract stays explicit (use the null - * bitmap instead). An empty array is accepted on the wire but QuestDB's - * BINARY storage uses the same NULL sentinel for zero-length and absent - * values, so an empty payload round-trips as NULL on read. + * bitmap instead). An empty array is accepted and remains a present, + * zero-length BINARY value, distinct from an omitted/null value. */ @Override public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { checkNotClosed(); checkTableSelected(); - if (value == null) { - throw new LineSenderException( - "BINARY value cannot be null; mark the row null via the null bitmap instead"); - } try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, value); + return this; + } + if (value == null) { + throw new LineSenderException( + "BINARY value cannot be null; mark the row null via the null bitmap instead"); + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BINARY, true); if (col != null) { col.addBinary(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1141,11 +1216,20 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, byte[] value) { public QwpWebSocketSender binaryColumn(CharSequence columnName, DirectByteSlice slice) { checkNotClosed(); checkTableSelected(); - if (slice == null) { - throw new LineSenderException( - "BINARY slice cannot be null; mark the row null via the null bitmap instead"); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, slice); + return this; + } + if (slice == null) { + throw new LineSenderException( + "BINARY slice cannot be null; mark the row null via the null bitmap instead"); + } + return binaryColumn(columnName, slice.ptr(), slice.size()); + } catch (RuntimeException | Error e) { + throw onRowFailure(e); } - return binaryColumn(columnName, slice.ptr(), slice.size()); } /** @@ -1158,13 +1242,17 @@ public QwpWebSocketSender binaryColumn(CharSequence columnName, long ptr, long l checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.binaryColumn(columnName, ptr, len); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BINARY, true); if (col != null) { col.addBinary(ptr, len); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1174,13 +1262,17 @@ public QwpWebSocketSender boolColumn(CharSequence columnName, boolean value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.boolColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BOOLEAN, false); if (col != null) { col.addBoolean(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1201,13 +1293,17 @@ public QwpWebSocketSender byteColumn(CharSequence columnName, byte value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.byteColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_BYTE, false); if (col != null) { col.addByte(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1234,13 +1330,17 @@ public QwpWebSocketSender charColumn(CharSequence columnName, char value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.charColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_CHAR, false); if (col != null) { col.addShort((short) value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1570,13 +1670,17 @@ public Sender decimalColumn(CharSequence name, Decimal64 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL64, true); if (col != null) { col.addDecimal64(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1587,13 +1691,17 @@ public Sender decimalColumn(CharSequence name, Decimal128 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL128, true); if (col != null) { col.addDecimal128(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1604,13 +1712,17 @@ public Sender decimalColumn(CharSequence name, Decimal256 value) { if (value == null || value.isNull()) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1621,14 +1733,18 @@ public Sender decimalColumn(CharSequence name, CharSequence value) { if (value == null || value.length() == 0) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.decimalColumn(name, value, currentDecimal256); + return this; + } currentDecimal256.ofString(value); QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DECIMAL256, true); if (col != null) { col.addDecimal256(currentDecimal256); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1639,13 +1755,17 @@ public Sender doubleArray(@NotNull CharSequence name, double[] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, values); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1656,13 +1776,17 @@ public Sender doubleArray(@NotNull CharSequence name, double[][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, values); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1673,13 +1797,17 @@ public Sender doubleArray(@NotNull CharSequence name, double[][][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, values); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1690,13 +1818,17 @@ public Sender doubleArray(CharSequence name, DoubleArray array) { if (array == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleArray(name, array); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_DOUBLE_ARRAY, true); if (col != null) { col.addDoubleArray(array); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1706,13 +1838,17 @@ public QwpWebSocketSender doubleColumn(CharSequence columnName, double value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.doubleColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_DOUBLE, true); if (col != null) { col.addDouble(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1728,13 +1864,17 @@ public QwpWebSocketSender floatColumn(CharSequence columnName, float value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.floatColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_FLOAT, true); if (col != null) { col.addFloat(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1869,18 +2009,22 @@ public boolean drain(long timeoutMillis) { public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int precisionBits) { checkNotClosed(); checkTableSelected(); - if (precisionBits < 1 || precisionBits > 60) { - throw new LineSenderException( - "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); - } try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.geoHashColumn(columnName, bits, precisionBits); + return this; + } + if (precisionBits < 1 || precisionBits > 60) { + throw new LineSenderException( + "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_GEOHASH, true); if (col != null) { col.addGeoHash(maskGeoHashBits(bits, precisionBits), precisionBits); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -1903,6 +2047,15 @@ public QwpWebSocketSender geoHashColumn(CharSequence columnName, long bits, int public QwpWebSocketSender geoHashColumn(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.geoHashColumn(columnName, value); + return this; + } + } catch (RuntimeException | Error e) { + throw onRowFailure(e); + } if (value == null) { throw new LineSenderException( "GEOHASH string cannot be null; mark the row null via the null bitmap instead"); @@ -2283,13 +2436,17 @@ public QwpWebSocketSender intColumn(CharSequence columnName, int value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.intColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_INT, true); if (col != null) { col.addInt(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2312,13 +2469,17 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, int address) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.ipv4Column(columnName, address); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_IPv4, true); if (col != null) { col.addIPv4(address); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2357,6 +2518,15 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre return this; } checkTableSelected(); + try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.ipv4Column(columnName, address); + return this; + } + } catch (RuntimeException | Error e) { + throw onRowFailure(e); + } if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { throw new LineSenderException( "invalid IPv4 address: NULL sentinel inputs are rejected" @@ -2374,6 +2544,12 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre /** * Adds a LONG256 column value to the current row. + *

+ * In schema-aware mode, setting all four limbs to {@link Long#MIN_VALUE} + * writes SQL NULL. Any other limb combination, including a partial match, + * is a value. Legacy mode preserves its existing behavior: the four limbs + * are emitted as native LONG256 wire data and server-side null/conversion + * handling applies. * * @param columnName the column name * @param l0 the least significant 64 bits @@ -2382,17 +2558,22 @@ public QwpWebSocketSender ipv4Column(CharSequence columnName, CharSequence addre * @param l3 the most significant 64 bits * @return this sender for method chaining */ + @Override public QwpWebSocketSender long256Column(CharSequence columnName, long l0, long l1, long l2, long l3) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.long256Column(columnName, l0, l1, l2, l3); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG256, true); if (col != null) { col.addLong256(l0, l1, l2, l3); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2403,13 +2584,18 @@ public Sender longArray(@NotNull CharSequence name, long[] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + // LONG_ARRAY is unimplemented; a schema refresh cannot make it valid. + throw onRowFailure(e); } return this; } @@ -2420,13 +2606,17 @@ public Sender longArray(@NotNull CharSequence name, long[][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2437,13 +2627,17 @@ public Sender longArray(@NotNull CharSequence name, long[][][] values) { if (values == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(values); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2454,13 +2648,17 @@ public Sender longArray(@NotNull CharSequence name, LongArray array) { if (array == null) return this; checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.unsupportedColumn(name, "LONG_ARRAY"); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(name, QwpConstants.TYPE_LONG_ARRAY, true); if (col != null) { col.addLongArray(array); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2470,13 +2668,17 @@ public QwpWebSocketSender longColumn(CharSequence columnName, long value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.longColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_LONG, true); if (col != null) { col.addLong(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2774,13 +2976,17 @@ public QwpWebSocketSender shortColumn(CharSequence columnName, short value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.shortColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SHORT, false); if (col != null) { col.addShort(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2890,13 +3096,17 @@ public QwpWebSocketSender stringColumn(CharSequence columnName, CharSequence val checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.stringColumn(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_VARCHAR, true); if (col != null) { col.addString(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2906,13 +3116,17 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.symbol(columnName, value); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_SYMBOL, true); if (col != null) { col.addSymbol(value); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2920,33 +3134,34 @@ public QwpWebSocketSender symbol(CharSequence columnName, CharSequence value) { @Override public QwpWebSocketSender table(CharSequence tableName) { checkNotClosed(); - // Fast path: if table name matches current, skip hashmap lookup - if (currentTableName != null && currentTableBuffer != null && Chars.equals(tableName, currentTableName)) { - return this; - } - // Prevent switching tables while a row is in progress - if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { - throw new LineSenderException("cannot switch tables while row is in progress" - + " [currentTable=").put(currentTableName).put(']'); - } - // Table changed - invalidate cached column references - validateTableName(tableName); - cachedTimestampColumn = null; - cachedTimestampNanosColumn = null; - currentTableBuffer = tableBuffers.get(tableName); - if (currentTableBuffer != null) { - currentTableName = currentTableBuffer.getTableName(); - } else { - currentTableName = tableName.toString(); - currentTableBuffer = new QwpTableBuffer(currentTableName, this); - tableBuffers.put(currentTableName, currentTableBuffer); - } - // Re-snap so sendRow()'s delta math is anchored to this table's - // current byte count. The prior current table's bytes already match - // its last-snapped value (the in-progress-row guard above ensures - // no column setters ran on it since the last consistency point). - currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); - // Both modes accumulate rows until flush + if (currentTableName == null || !Chars.equals(tableName, currentTableName)) { + // Prevent switching tables while a row is in progress. + if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { + throw new LineSenderException("cannot switch tables while row is in progress" + + " [currentTable=").put(currentTableName).put(']'); + } + validateTableName(tableName); + // Table changed - invalidate cached column references. + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + currentTableBuffer = tableBuffers.get(tableName); + if (currentTableBuffer != null) { + currentTableName = currentTableBuffer.getTableName(); + } else { + currentTableName = tableName.toString(); + currentTableBuffer = new QwpTableBuffer(currentTableName, this); + tableBuffers.put(currentTableName, currentTableBuffer); + } + // Re-snap so sendRow()'s delta math is anchored to this table's + // current byte count. The prior current table's bytes already match + // its last-snapped value (the in-progress-row guard above ensures + // no column setters ran on it since the last consistency point). + currentTableBufferSnapshotBytes = currentTableBuffer.getBufferedBytes(); + } + // Resolve schema at the explicit table boundary. Setters retain the + // same lookup as a fallback because the selected table also starts + // implicit rows after completion, cancellation, and batch flushes. + bindingForEffectiveWrite(); return this; } @@ -2955,6 +3170,11 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.timestampColumn(columnName, value, unit); + return this; + } if (unit == ChronoUnit.NANOS) { QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_TIMESTAMP_NANOS, true); if (col != null) { @@ -2968,8 +3188,7 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, long value, C } } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -2979,14 +3198,18 @@ public QwpWebSocketSender timestampColumn(CharSequence columnName, Instant value checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.timestampColumn(columnName, value); + return this; + } long micros = value.getEpochSecond() * 1_000_000L + value.getNano() / 1000L; QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_TIMESTAMP, true); if (col != null) { col.addLong(micros); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3003,13 +3226,17 @@ public QwpWebSocketSender uuidColumn(CharSequence columnName, long lo, long hi) checkNotClosed(); checkTableSelected(); try { + QwpSchemaBinding binding = bindingForEffectiveWrite(); + if (binding != null) { + binding.uuidColumn(columnName, lo, hi); + return this; + } QwpTableBuffer.ColumnBuffer col = currentTableBuffer.getOrCreateColumn(columnName, QwpConstants.TYPE_UUID, true); if (col != null) { col.addUuid(hi, lo); } } catch (RuntimeException | Error e) { - rollbackRow(); - throw e; + throw onRowFailure(e); } return this; } @@ -3372,6 +3599,10 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo newClient.setQwpMaxVersion(QwpConstants.VERSION); newClient.setQwpClientId(QwpConstants.CLIENT_ID); newClient.setQwpRequestDurableAck(requestDurableAck); + // OFF still negotiates when this stream must replay schema frames. + if (schemaMode != Sender.SchemaMode.OFF || ctx.requiresSchema()) { + newClient.requestQwpSchema(); + } newClient.setConnectTimeout(effectiveConnectTimeoutMs(background, connectTimeoutMs)); if (cancellation != null) { // Publish the client we are about to block on so a @@ -3499,6 +3730,22 @@ private WebSocketClient connectWalk(ReconnectSupplier ctx, CursorWebSocketSendLo // rethrow. close() is CAS-gated, so re-closing after the // durable-ack arm's own close is a no-op. try { + if (ctx.requiresSchema() && !newClient.isQwpSchemaEnabled()) { + newClient.close(); + hostTracker.recordRoleReject(idx, false, !background); + QwpSchemaCapabilityMismatchException schemaErr = new QwpSchemaCapabilityMismatchException(); + // Schema-backed bytes may never enter the durable-ack + // quarantine path, so schema evidence outranks every + // other capability gap in this sweep. + terminalUpgradeError = schemaErr; + lastError = schemaErr; + if (!background) { + dispatchConnectionEvent(SenderConnectionEvent.Kind.ENDPOINT_ATTEMPT_FAILED, + ep.host, ep.port, null, SenderConnectionEvent.NO_PORT, + attemptNumber, roundSeq, schemaErr); + } + continue; + } if (requestDurableAck && !newClient.isServerDurableAckEnabled()) { newClient.close(); hostTracker.recordRoleReject(idx, false, !background); @@ -3917,7 +4164,8 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn LOG.error("Error in close-drain-waiting test hook: {}", String.valueOf(t)); } } - long deadlineNanos = System.nanoTime() + closeFlushTimeoutMillis * 1_000_000L; + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(closeFlushTimeoutMillis); restoreInterrupt[0] |= Thread.interrupted(); try { while (cursorEngine.ackedFsn() < target) { @@ -3937,7 +4185,7 @@ private void drainOnClose(boolean errorOwnedByCustomHandler, boolean[] restoreIn } else { cursorSendLoop.checkError(); } - if (System.nanoTime() >= deadlineNanos) { + if (System.nanoTime() - startNanos >= budgetNanos) { long acked = cursorEngine.ackedFsn(); // Name the outage the I/O thread is riding out, when there is one. A // foreground sender now retries endpoint-policy rejections indefinitely, @@ -4017,7 +4265,13 @@ private void ensureConnected() { connectionDispatcher = new SenderConnectionDispatcher( connectionListener, connectionListenerInboxCapacity); } + // A recovered deferred-only tail is an aborted transaction that never + // reaches the wire. Retire it locally before the capability check, as + // BackgroundDrainer does, so its schema frames cannot demand a + // schema-capable peer and block startup against a legacy server. + cursorEngine.retireRecoveredOrphanTailIfReady(); CursorWebSocketSendLoop.ReconnectFactory reconnectFactory = newReconnectFactory(); + reconnectFactory.setSchemaRequired(cursorEngine.requiresSchema()); switch (initialConnectMode) { case SYNC: client = CursorWebSocketSendLoop.connectWithRetry( @@ -4094,6 +4348,10 @@ private void ensureConnected() { // the loop no longer fires a terminal budget-exhaustion event -- it // retries indefinitely.) cursorSendLoop.setConnectionDispatcher(connectionDispatcher); + // AUTO falls back to the legacy contract when the wire drops under a + // lookup, so the lookup must fail right away rather than ride the + // reconnect. STRICT keeps it pending for the replacement connection. + cursorSendLoop.setAbandonSchemaLookupOnDisconnect(schemaMode == Sender.SchemaMode.AUTO); cursorSendLoop.start(); } catch (Throwable t) { // start() (or dispatcher construction) failed after cursorSendLoop was @@ -4152,6 +4410,189 @@ private void ensureConnected() { connected = true; } + /** + * Selects the wire contract at an explicit table boundary or before the first + * effective value of an implicit row. + *

+ * A table pins one schema snapshot per batch. Once the table holds pending rows, + * later rows of the same batch validate against that snapshot without consulting + * the cache; the first row after a flush or reset adopts whatever the cache holds + * by then, which ACK schema feedback keeps current. A rejection that a column + * type change could explain evicts the snapshot; see {@link #onRowFailure}. + *

+ * When no snapshot is pinned, the configured {@link Sender.SchemaMode} decides + * between a schema lookup and the legacy contract; see + * {@link #resolveSchemaForCurrentTable()}. + */ + private QwpSchemaBinding bindingForEffectiveWrite() { + QwpSchemaBinding pinned = currentTableBuffer.getSchemaBinding(); + if (currentTableBuffer.hasInProgressRow()) { + return pinned; + } + ensureConnected(); + if (pinned != null && currentTableBuffer.getRowCount() > 0) { + return pinned; + } + // A batch has one wire contract, because a QWP frame has one wire family. + // While the pending batch holds legacy rows, later rows stay legacy even if + // a schema has become obtainable since; the first row after the flush + // consults the schema again. + if (pendingRowCount > 0 && isPendingBatchLegacy) { + return selectLegacyContract(); + } + QwpSchemaResponse latest = resolveSchemaForCurrentTable(); + if (latest == null) { + return selectLegacyContract(); + } + if (pinned != null + && pinned.getTableId() == latest.getTableId() + && pinned.getMetadataVersion() == latest.getMetadataVersion()) { + return pinned; + } + return installSchemaBinding(latest); + } + + /** + * Resolves the current table's schema under the configured + * {@link Sender.SchemaMode}, or returns {@code null} when the row must take the + * legacy contract: the mode is OFF, the server negotiated legacy QWP, or, in + * AUTO, the schema is not obtainable. + *

+ * STRICT waits for the first upgrade and then for the lookup within one + * {@code schemaWaitMillis} budget and lets a lookup failure propagate. AUTO + * never waits on a dead wire: before the first upgrade it takes the legacy + * contract outright, while the wire is down it consults only the cache, and + * a lookup that ends in {@code SCHEMA_UNAVAILABLE} (a drop under it, a timeout, + * a server-side unavailability) or {@code UNSUPPORTED_FEATURE} (an oversized + * or unsupported schema response) selects the legacy contract instead of failing + * the row. Access denial still fails the row. + */ + private QwpSchemaResponse resolveSchemaForCurrentTable() { + QwpSchemaResponse schema = lookUpSchemaForCurrentTable(); + if (schema == null || schema.getResult() != QwpSchemaProtocol.RESULT_TOO_LARGE) { + return schema; + } + // The coordinator caches TOO_LARGE, so this costs no round trip per batch. + if (schemaMode == Sender.SchemaMode.STRICT) { + throw new LineSenderSchemaException(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + "schema lookup failed [reason=UNSUPPORTED_FEATURE, table=" + currentTableName + + ", detail=schema could not be returned within the discovery response limit]"); + } + return null; + } + + private QwpSchemaResponse lookUpSchemaForCurrentTable() { + if (schemaMode == Sender.SchemaMode.OFF) { + return null; + } + final long startNanos = System.nanoTime(); + final long budgetNanos = TimeUnit.MILLISECONDS.toNanos(schemaWaitMillis); + if (schemaMode == Sender.SchemaMode.STRICT) { + if (!cursorSendLoop.awaitInitialSchemaMode(remainingSchemaMillis(startNanos, budgetNanos))) { + return null; + } + // STRICT writes no rows to a table it cannot describe, so no ACK feedback + // ever reports that the table shrank: ask again instead of trusting a + // cached TOO_LARGE. AUTO keeps the cache hit; its legacy rows earn feedback. + QwpSchemaResponse cached = cursorSendLoop.peekSchema(currentTableBuffer.getSchemaKey()); + if (cached != null && cached.getResult() == QwpSchemaProtocol.RESULT_TOO_LARGE) { + return cursorSendLoop.refreshSchema(currentTableBuffer.getSchemaKey(), remainingSchemaMillis(startNanos, budgetNanos)); + } + return cursorSendLoop.resolveSchema(currentTableBuffer.getSchemaKey(), remainingSchemaMillis(startNanos, budgetNanos)); + } + if (!cursorSendLoop.hasEverConnected()) { + return null; + } + if (!cursorSendLoop.isWireUp()) { + return cursorSendLoop.peekSchema(currentTableBuffer.getSchemaKey()); + } + if (!cursorSendLoop.isSchemaEnabled()) { + return null; + } + try { + return cursorSendLoop.resolveSchema(currentTableBuffer.getSchemaKey(), remainingSchemaMillis(startNanos, budgetNanos)); + } catch (LineSenderSchemaException e) { + if (e.getReason() == LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE + || e.getReason() == LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE) { + return null; + } + throw e; + } + } + + /** + * Commits the current table's next row to the legacy contract. A pending + * schema batch cannot take a legacy row, because a QWP frame has one wire + * family, so it is published first; a buffer still bound to an earlier batch's + * snapshot is unbound so the legacy setters lay out its columns afresh. + */ + private QwpSchemaBinding selectLegacyContract() { + if (pendingRowCount > 0 && !isPendingBatchLegacy) { + flushPendingRows(transactional); + } + if (currentTableBuffer.getSchemaBinding() != null) { + unbindCurrentTableBuffer(); + } + return null; + } + + /** + * Binds the row-less current table buffer to {@code latest}. A buffer that has + * been bound before, or that a flushed legacy batch left with a column layout + * ({@link QwpTableBuffer#reset()} keeps columns), cannot take a new binding in + * place; {@link QwpTableBuffer#clear()} drops both before rebinding. + */ + private QwpSchemaBinding installSchemaBinding(QwpSchemaResponse latest) { + if (currentTableBuffer.getSchemaBinding() != null || currentTableBuffer.getColumnCount() != 0) { + unbindCurrentTableBuffer(); + } + return new QwpSchemaBinding(currentTableBuffer, latest); + } + + /** + * Drops the row-less current table buffer's binding and column layout so the + * next contract lays it out afresh. + */ + private void unbindCurrentTableBuffer() { + assert currentTableBuffer.getRowCount() == 0 : "table buffer unbound under pending rows"; + currentTableBuffer.clear(); + currentTableBufferSnapshotBytes = 0; + cachedTimestampColumn = null; + cachedTimestampNanosColumn = null; + } + + // Single home of the row-failure policy: roll the whole row back, then report + // the failure as raised. A schema rejection is final for the batch's snapshot; + // the sender never looks up fresh metadata from a failed setter. When the + // snapshot type of a known column rejected the value, the column may have + // changed type since, and no frame may ever reach the server to earn feedback. + // The sender then evicts the snapshot, so the next batch looks the table up. + private RuntimeException onRowFailure(Throwable e) { + QwpSchemaBinding binding = currentTableBuffer != null ? currentTableBuffer.getSchemaBinding() : null; + if (binding != null && binding.hasTargetTypeConflict()) { + binding.clearTargetTypeConflict(); + CursorWebSocketSendLoop loop = cursorSendLoop; + if (loop != null) { + loop.evictSchema(currentTableBuffer.getTableName(), binding.getTableId(), binding.getMetadataVersion()); + } + } + rollbackRow(); + if (e instanceof Error) { + throw (Error) e; + } + return (RuntimeException) e; + } + + /** + * Milliseconds left of a saturated nanosecond budget, rounded up. Comparing + * elapsed time against the budget cannot wrap, unlike an absolute deadline, + * so a budget of Long.MAX_VALUE millis means no limit. + */ + private static long remainingSchemaMillis(long startNanos, long budgetNanos) { + long remaining = budgetNanos - (System.nanoTime() - startNanos); + return remaining <= 0 ? 0 : 1L + (remaining - 1L) / 1_000_000L; + } + private void ensureNoInProgressRow() { if (currentTableBuffer != null && currentTableBuffer.hasInProgressRow()) { throw new LineSenderException( @@ -4192,6 +4633,10 @@ private void flushPendingRows(boolean deferCommit) { firstPendingRowTimeNanos = 0; return; } + if (tableCount > 0xffff) { + throw new LineSenderException("too many table blocks in one QWP message [count=") + .put(tableCount).put("]"); + } if (LOG.isDebugEnabled()) { LOG.debug("Flushing pending rows [count={}, tables={}, defer={}]", pendingRowCount, tableCount, deferCommit); @@ -4309,8 +4754,14 @@ && splitFramesFit(cap, currentBatchMaxSymbolId)) { */ private int encodeCombinedFrame(int tableCount, boolean deferCommit, int deltaBaseline) { encoder.setDeferCommit(deferCommit); - encoder.beginMessage(tableCount, globalSymbolDictionary, - deltaBaseline, currentBatchMaxSymbolId); + boolean schemaMessage = flushTableBuffers.getQuick(0).getSchemaBinding() != null; + if (schemaMessage) { + encoder.beginSchemaMessage(tableCount, globalSymbolDictionary, + deltaBaseline, currentBatchMaxSymbolId); + } else { + encoder.beginMessage(tableCount, globalSymbolDictionary, + deltaBaseline, currentBatchMaxSymbolId); + } splitFrameBodyBytes.clear(); int combinedBodyStart = encoder.getBuffer().getPosition(); int bodyStart = combinedBodyStart; @@ -4322,7 +4773,15 @@ private int encodeCombinedFrame(int tableCount, boolean deferCommit, int deltaBa flushTableNames.getQuick(i), tableBuffer.getRowCount(), currentBatchMaxSymbolId); } - encoder.addTable(tableBuffer); + QwpSchemaBinding binding = tableBuffer.getSchemaBinding(); + if ((binding != null) != schemaMessage) { + throw new IllegalStateException("cannot mix legacy and schema table blocks in one frame"); + } + if (binding != null) { + encoder.addSchemaTable(tableBuffer, binding.getTableId(), binding.getMetadataVersion()); + } else { + encoder.addTable(tableBuffer); + } int bodyEnd = encoder.getBuffer().getPosition(); splitFrameBodyBytes.add(bodyEnd - bodyStart); bodyStart = bodyEnd; @@ -5215,6 +5674,9 @@ private void sendRow() { if (pendingRowCount == 0) { firstPendingRowTimeNanos = System.nanoTime(); + // The batch's first row selects its wire contract; see + // bindingForEffectiveWrite. + isPendingBatchLegacy = currentTableBuffer.getSchemaBinding() == null; } pendingRowCount++; @@ -5354,6 +5816,8 @@ private final class ReconnectSupplier implements CursorWebSocketSendLoop.Reconne */ private final java.util.function.BooleanSupplier abortCheck; private final String abortMessage; + // Each factory serves one engine; other streams can still use legacy peers. + private volatile boolean isSchemaRequired; private int previousIdx = -1; private ReconnectSupplier() { @@ -5390,6 +5854,15 @@ boolean isAborted() { : (cursorSendLoop == null ? closed : !cursorSendLoop.isRunning()); } + boolean requiresSchema() { + return isSchemaRequired; + } + + @Override + public void setSchemaRequired(boolean isRequired) { + isSchemaRequired = isRequired; + } + @Override public WebSocketClient reconnect() { return buildAndConnect(this, null); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java index 81d59d28f..11c9df07a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java @@ -24,7 +24,11 @@ package io.questdb.client.cutlass.qwp.client; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.std.LongList; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.std.Utf8SequenceObjHashMap; @@ -32,6 +36,9 @@ import io.questdb.client.std.str.Utf8s; import org.jetbrains.annotations.TestOnly; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; /** @@ -63,6 +70,9 @@ */ public class WebSocketResponse { + public static final int SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL = 0xC0; + public static final int SCHEMA_FEEDBACK_MODE_MASK = 0xC0; + public static final int SCHEMA_FEEDBACK_MODE_UPDATES = 0x80; public static final int MAX_ERROR_MESSAGE_LENGTH = 1024; public static final int MIN_DURABLE_ACK_SIZE = 3; // status + tableCount public static final int MIN_ERROR_RESPONSE_SIZE = 11; // status + sequence + error length @@ -102,9 +112,12 @@ public class WebSocketResponse { private final Utf8SequenceObjHashMap tableNameCache = new Utf8SequenceObjHashMap<>(); private final ObjList tableNames = new ObjList<>(); private final LongList tableSeqTxns = new LongList(); + private final ObjList schemaUpdateTableNames = new ObjList<>(); + private final ObjList schemaUpdates = new ObjList<>(); private String errorMessage; private int errorMessageUtf8Length; private long sequence; + private boolean schemaInvalidation; private byte status; public WebSocketResponse() { @@ -249,6 +262,26 @@ public long getTableSeqTxn(int index) { return tableSeqTxns.getQuick(index); } + public int getSchemaUpdateCount() { + return schemaUpdates.size(); + } + + public QwpSchemaResponse getSchemaUpdate(int index) { + return schemaUpdates.getQuick(index); + } + + public String getSchemaUpdateTableName(int index) { + return schemaUpdateTableNames.getQuick(index); + } + + public boolean hasSchemaUpdates() { + return schemaUpdates.size() > 0; + } + + public boolean isSchemaInvalidation() { + return schemaInvalidation; + } + /** * Returns true when this is a per-table durable-upload ACK (STATUS_DURABLE_ACK). */ @@ -271,14 +304,38 @@ public boolean isSuccess() { * @return true if successfully parsed, false if not enough data */ public boolean readFrom(long ptr, int length) { + return readFrom(ptr, length, false); + } + + /** + * Reads a response, accepting schema feedback only on a connection that + * negotiated schema support during its WebSocket upgrade. + */ + public boolean readFrom(long ptr, int length, boolean schemaNegotiated) { tableNames.clear(); tableSeqTxns.clear(); + schemaUpdateTableNames.clear(); + schemaUpdates.clear(); + schemaInvalidation = false; if (length < 1) { return false; } - status = Unsafe.getUnsafe().getByte(ptr); + int wireStatus = Unsafe.getUnsafe().getByte(ptr) & 0xff; + int feedbackMode = wireStatus & SCHEMA_FEEDBACK_MODE_MASK; + if (feedbackMode == 0x40 || (feedbackMode != 0 && !schemaNegotiated)) { + return false; + } + if (feedbackMode == SCHEMA_FEEDBACK_MODE_UPDATES && length > QwpSchemaProtocol.MAX_FRAME_SIZE) { + return false; + } + byte baseStatus = (byte) (wireStatus & ~SCHEMA_FEEDBACK_MODE_MASK); + if (baseStatus == STATUS_DURABLE_ACK && feedbackMode != 0) { + return false; + } + status = baseStatus; + int baseEnd; if (status == STATUS_OK) { if (length < MIN_OK_RESPONSE_SIZE) { @@ -287,7 +344,12 @@ public boolean readFrom(long ptr, int length) { sequence = Unsafe.getUnsafe().getLong(ptr + 1); errorMessage = null; errorMessageUtf8Length = -1; - return readTableEntries(ptr + 9, length - 9); + baseEnd = readTableEntries(ptr + 9, length - 9, feedbackMode != 0); + if (baseEnd < 0) { + return false; + } + baseEnd += 9; + return readFeedback(ptr, length, baseEnd, feedbackMode); } if (status == STATUS_DURABLE_ACK) { @@ -297,7 +359,7 @@ public boolean readFrom(long ptr, int length) { sequence = -1; errorMessage = null; errorMessageUtf8Length = -1; - return readTableEntries(ptr + 1, length - 1); + return readTableEntries(ptr + 1, length - 1, false) >= 0; } // Error response @@ -322,7 +384,8 @@ public boolean readFrom(long ptr, int length) { errorMessage = null; errorMessageUtf8Length = 0; } - return true; + baseEnd = offset + msgLen; + return readFeedback(ptr, length, baseEnd, feedbackMode); } /** @@ -391,15 +454,15 @@ public int writeTo(long ptr) { // entries, empty table names, or trailing garbage. On false, tableNames / // tableSeqTxns may hold partial state, but the caller (readFrom) clears // both lists at the start of every call so partial state never leaks. - private boolean readTableEntries(long ptr, int remaining) { + private int readTableEntries(long ptr, int remaining, boolean allowTrailing) { if (remaining < 2) { - return false; + return -1; } int tableCount = Unsafe.getUnsafe().getShort(ptr) & 0xFFFF; int offset = 2; for (int i = 0; i < tableCount; i++) { if (remaining < offset + 2) { - return false; + return -1; } int nameLen = Unsafe.getUnsafe().getShort(ptr + offset) & 0xFFFF; offset += 2; @@ -407,7 +470,7 @@ private boolean readTableEntries(long ptr, int remaining) { // table name is never zero bytes, and accepting empty names would // let a misbehaving server poison the per-table tracker with "" entries. if (nameLen == 0 || remaining < offset + nameLen + 8) { - return false; + return -1; } long nameLo = ptr + offset; long nameHi = nameLo + nameLen; @@ -417,7 +480,87 @@ private boolean readTableEntries(long ptr, int remaining) { tableNames.add(internTableName(nameLo, nameHi)); tableSeqTxns.add(seqTxn); } - return remaining == offset; + return allowTrailing || remaining == offset ? offset : -1; + } + + private boolean readFeedback(long ptr, int length, int offset, int mode) { + if (mode == 0) { + return offset == length; + } + if (mode == SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL) { + if (offset != length) { + return false; + } + schemaInvalidation = true; + return true; + } + if (length - offset < 2) { + return false; + } + int count = Unsafe.getUnsafe().getShort(ptr + offset) & 0xffff; + offset += 2; + if (count == 0) { + return false; + } + // Each entry needs nameLen + one name byte + payloadLen + the shortest + // schema payload. Reject hostile counts before sizing collections. + if (count > (length - offset) / 17) { + return false; + } + ObjList names = new ObjList<>(count); + ObjList responses = new ObjList<>(count); + LowerCaseCharSequenceIntHashMap seen = new LowerCaseCharSequenceIntHashMap(count); + try { + for (int i = 0; i < count; i++) { + if (length - offset < 2) { + return false; + } + int nameLength = Unsafe.getUnsafe().getShort(ptr + offset) & 0xffff; + offset += 2; + if (nameLength == 0 || nameLength > QwpSchemaProtocol.MAX_NAME_UTF8_LENGTH || length - offset < nameLength + 4) { + return false; + } + String name = decodeStrictUtf8(ptr + offset, nameLength); + if (!TableUtils.isValidTableName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH) || !seen.put(name, i)) { + return false; + } + offset += nameLength; + long payloadLength = Unsafe.getUnsafe().getInt(ptr + offset) & 0xffffffffL; + offset += 4; + if (payloadLength > QwpSchemaProtocol.MAX_FRAME_SIZE || payloadLength > length - offset) { + return false; + } + QwpSchemaResponse response = QwpSchemaProtocol.decodeFeedbackPayload(ptr + offset, (int) payloadLength); + offset += (int) payloadLength; + names.add(name); + responses.add(response); + } + } catch (IllegalArgumentException e) { + return false; + } + if (offset != length) { + return false; + } + for (int i = 0; i < names.size(); i++) { + schemaUpdateTableNames.add(names.getQuick(i)); + schemaUpdates.add(responses.getQuick(i)); + } + return true; + } + + private static String decodeStrictUtf8(long address, int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException("invalid UTF-8 feedback table name", e); + } } private String internTableName(long lo, long hi) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index facd872bb..5833ffbd2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpCredentialUnavailableException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; @@ -407,6 +408,10 @@ public static long dynamicCredentialAuthDwellNanos(long reconnectMaxDurationMill * {@link #outcome} has been set to FAILED or STOPPED */ public WebSocketClient connectWithDurableAckRetry() { + return connectWithDurableAckRetry(null); + } + + private WebSocketClient connectWithDurableAckRetry(CursorSendEngine schemaEngine) { // run() already set runnerThread; setting it again here is a no-op // on that path but wires up direct callers so requestStop() // can unpark them too. @@ -477,7 +482,16 @@ public WebSocketClient connectWithDurableAckRetry() { // to the deadline (which would otherwise busy-loop once past it). boolean boundedByBudget = false; try { - return clientFactory.reconnect(); + if (schemaEngine != null) { + clientFactory.setSchemaRequired(schemaEngine.requiresSchema()); + } + WebSocketClient connected = clientFactory.reconnect(); + if (connected != null && schemaEngine != null && schemaEngine.requiresSchema() + && !connected.isQwpSchemaEnabled()) { + connected.close(); + throw new QwpSchemaCapabilityMismatchException(); + } + return connected; } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // A non-421 upgrade reject, and a 401/403 against a CONSTANT credential, are genuinely // non-retriable across the cluster: waiting will not fix them, so quarantine immediately @@ -589,6 +603,16 @@ public WebSocketClient connectWithDurableAckRetry() { slotPath, roleRejectAttempts); lastReplicaWarnNanos = nowWarn; } + } catch (QwpSchemaCapabilityMismatchException e) { + // Extended bytes cannot be downgraded, discarded or quarantined. + // Keep the slot and retry until a supporting endpoint returns. + capabilityGapAttempts = 0; + capabilityGapElapsedNanos = 0L; + lastCapabilityGapNanos = 0L; + firstDynamicCredentialAuthFailureNanos = 0L; + lastErrorMessage = e.getMessage(); + LOG.warn("drainer slot {}: schema framing unavailable, retaining backlog and retrying", + slotPath); } catch (QwpDurableAckMismatchException e) { // Genuine cluster-wide durable-ack CAPABILITY gap: a server // upgraded but does not advertise durable ack. Unlike a role @@ -1072,7 +1096,7 @@ public void run() { // poll of a partially-drained slot read as progress and hand back a budget the initial connect // had legitimately spent. ackProgressWatermark = engine.ackedFsn(); - client = connectWithDurableAckRetry(); + client = connectWithDurableAckRetry(engine); if (client == null) { // outcome already set (FAILED or STOPPED); markFailed sentinel // already dropped on the FAILED path. @@ -1181,7 +1205,8 @@ public void run() { if (t.getCause() instanceof Error) { throw (Error) t.getCause(); } - if (loop.capabilityGapTerminal() != null || loop.authTerminal() != null) { + if (loop.capabilityGapTerminal() != null + || loop.authTerminal() != null) { // The I/O thread publishes a durable ack before it latches a later terminal. // That publication can land after the poll at the top of this iteration but // before checkError() observes the terminal. Re-read here so recycling the wire @@ -1243,7 +1268,7 @@ public void run() { } catch (Throwable ignored) { } loop = null; - client = connectWithDurableAckRetry(); + client = connectWithDurableAckRetry(engine); if (client == null) { // outcome already set (FAILED after budget // exhaustion, or STOPPED); sentinel handled. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 66a0635ea..1215446d2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; @@ -174,6 +175,12 @@ public final class CursorSendEngine implements QuietCloseable { // because both producer seeding and every recycled send loop need the same // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. private final RecoveredFrameAnalysis recoveredFrameAnalysis; + // FSN of the newest FLAG_SCHEMA frame in the ring; -1 when none. A schema + // requirement exists only while this frame is unacked, so a peer without + // schema support becomes usable again once the server acks it. Written by + // the single producer thread (and by the constructor from recovery), read + // by the I/O thread. + private volatile long maxSchemaFsn = -1L; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -820,6 +827,9 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.watermark = watermarkInProgress; this.persistedSymbolDict = persistedDictInProgress; this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; + if (recoveredFrameAnalysisInProgress != null) { + this.maxSchemaFsn = recoveredFrameAnalysisInProgress.maxSchemaFsn(); + } } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -916,6 +926,7 @@ MmapSegment pinActiveSegment() { * is wedged" failures (server down, slow disk, etc.) from the user. */ public long appendBlocking(long payloadAddr, int payloadLen) { + trackSchemaFrame(payloadAddr, payloadLen); long fsn = ring.appendOrFsn(payloadAddr, payloadLen); if (fsn >= 0) return fsn; if (fsn == SegmentRing.PAYLOAD_TOO_LARGE) { @@ -959,6 +970,7 @@ public long appendBlocking(long payloadAddr, int payloadLen) { * {@code SegmentRing.BACKPRESSURE_*} / {@code PAYLOAD_*} sentinels. */ public long appendOrFsn(long payloadAddr, int payloadLen, long spinDeadlineNanos) { + trackSchemaFrame(payloadAddr, payloadLen); long fsn = ring.appendOrFsn(payloadAddr, payloadLen); if (fsn >= 0) { return fsn; @@ -983,6 +995,28 @@ public void checkDurability() { ring.checkDurability(); } + /** + * True while an unacked frame in the ring carries {@code FLAG_SCHEMA} and + * therefore must only reach a peer that negotiated schema framing. + */ + public boolean requiresSchema() { + return maxSchemaFsn > ring.ackedFsn(); + } + + private void trackSchemaFrame(long payloadAddr, int payloadLen) { + if (payloadAddr != 0 + && payloadLen >= QwpConstants.HEADER_SIZE + && io.questdb.client.std.Unsafe.getUnsafe().getInt(payloadAddr) == QwpConstants.MAGIC_MESSAGE + && (io.questdb.client.std.Unsafe.getUnsafe().getByte( + payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA) != 0) { + // Publish the capability requirement before ring.append publishes + // the frame to the I/O consumer. The single producer makes the + // next FSN exact; if the append fails, the frame that eventually + // takes this FSN keeps the requirement until it is acked. + maxSchemaFsn = ring.publishedFsn() + 1L; + } + } + @Override public void close() { close(true); @@ -1876,7 +1910,9 @@ public long recoveredOrphanTipFsn() { /** * Retires a recovered deferred tail once every frame below it is ACKed. * The operation is local and idempotent: no wire sequence ever referred - * to these aborted-transaction frames. + * to these aborted-transaction frames, so it needs no peer capability. + * Retiring schema-framed tail frames also drops the schema requirement + * they imposed. * * @return true if no orphan tail remains, false if lower frames still need * server ACKs diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6643bf036..dd984e464 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.LineSenderServerException; +import io.questdb.client.LineSenderSchemaException; import io.questdb.client.SenderConnectionEvent; import io.questdb.client.SenderError; import io.questdb.client.cutlass.http.client.WebSocketClient; @@ -35,11 +36,14 @@ import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpCredentialUnavailableException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; import io.questdb.client.std.MemoryTag; @@ -271,6 +275,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final CursorSendEngine engine; private final long parkNanos; private final ReconnectPolicy reconnectPolicy; + private final QwpSchemaCoordinator schemaCoordinator = new QwpSchemaCoordinator(); // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -431,6 +436,17 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // foreground sender's transient auth blip becomes a producer-fatal terminal -- the // exact failure this policy exists to prevent. private volatile boolean hasEverConnected; + // STRICT row policy remembers successful schema negotiation independently of replay. + private volatile boolean hasNegotiatedSchema; + // Producer-visible wire state. True from the moment swapClient installs a + // connection until the I/O thread enters the reconnect loop for it. The + // sender's schema contract selection reads it to decide between a lookup + // (wire up) and a cache-only probe (wire down). + private volatile boolean isWireUp; + // When true, connectLoop fails a pending schema lookup immediately instead of + // letting the replacement connection re-send it. Set by a sender whose schema + // mode can fall back to the legacy contract. + private volatile boolean shouldAbandonSchemaLookupOnDisconnect; // Cause of the outage the reconnect loop is currently riding out, or null once a // connect succeeds. Written by the I/O thread, read by the producer thread so a // backpressure or drain-timeout failure can NAME the reason the wire is not draining @@ -918,7 +934,9 @@ private CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // already know we reached the server at least once. ASYNC startup // hands null and lets the I/O thread connect — hasEverConnected // stays false until swapClient sees its first success. + this.hasNegotiatedSchema = client != null && client.isQwpSchemaEnabled(); this.hasEverConnected = client != null; + this.isWireUp = client != null; // Adopt the engine's recovered orphaned-deferred-tail range (if any). // See the field docs on orphanSkipStartFsn/orphanSkipTipFsn; the // BackgroundDrainer path builds its loop through this same @@ -1201,6 +1219,54 @@ public void checkError() { } } + /** + * Resolves a schema through this loop's I/O thread, returning a KNOWN, MISSING or TOO_LARGE response. + * This producer-side API permits one outstanding lookup. The timeout covers the entire + * operation, including connection and I/O; zero permits only an existing cache hit. + * Failures are reported as {@link LineSenderSchemaException}. This does not mutate rows, + * store-and-forward state, or acknowledgment watermarks. + */ + public QwpSchemaResponse resolveSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.resolve(tableName, timeoutMillis, false); + } + + /** + * Cache-only variant of {@link #resolveSchema}: the schema the coordinator + * already holds for the table, or {@code null}. Never sends a request and + * never waits, so a producer can call it while the wire is down. + */ + public QwpSchemaResponse peekSchema(CharSequence tableName) { + return schemaCoordinator.peek(tableName); + } + + /** + * Evicts the cached KNOWN snapshot of {@code tableName} if it still has the + * given identity, so the next batch that writes the table looks it up again. + */ + public void evictSchema(CharSequence tableName, int tableId, long metadataVersion) { + schemaCoordinator.evictKnown(tableName, tableId, metadataVersion); + } + + /** + * Selects what happens to a schema lookup in flight when the connection + * drops: {@code true} fails it immediately, {@code false} (the default) keeps + * it pending with its original deadline for the replacement connection. + */ + public void setAbandonSchemaLookupOnDisconnect(boolean abandon) { + this.shouldAbandonSchemaLookupOnDisconnect = abandon; + } + + /** + * Evicts the named cache entry at admission and resolves it again through the I/O thread. + * This producer-side API permits one outstanding lookup. The timeout covers the entire + * operation, including connection and I/O, and must leave time for a request. It returns + * a KNOWN, MISSING or TOO_LARGE response and reports failures as {@link LineSenderSchemaException}. + * This does not mutate rows, store-and-forward state, or acknowledgment watermarks. + */ + public QwpSchemaResponse refreshSchema(CharSequence tableName, long timeoutMillis) { + return schemaCoordinator.resolve(tableName, timeoutMillis, true); + } + /** * The typed rotating-credential auth terminal (401/403), or {@code null} if the loop's terminal * (if any) is a different failure class. Non-null only after {@link #checkError()} started @@ -1259,6 +1325,7 @@ public synchronized void close() { // is false and start's ioLoop will exit immediately) or entirely // after — the latch await is only skipped when the loop never ran. running = false; + schemaCoordinator.close(); Thread t = ioThread; // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets // freed on ioLoop's exit path. When t == null the loop never ran (start() @@ -1534,6 +1601,54 @@ public boolean hasEverConnected() { return hasEverConnected; } + public boolean isSchemaEnabled() { + WebSocketClient activeClient = client; + return activeClient != null && activeClient.isQwpSchemaEnabled(); + } + + /** + * Whether a connection is currently installed. False before the first + * upgrade completes and from the moment the I/O thread enters the reconnect + * loop until {@code swapClient} installs the replacement. + */ + public boolean isWireUp() { + return isWireUp; + } + + /** + * Waits for the first completed WebSocket upgrade and returns whether any + * installed connection has selected schema mode. A failed or still-pending + * handshake is never interpreted as a legacy peer. + */ + public boolean awaitInitialSchemaMode(long timeoutMillis) { + long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + long start = System.nanoTime(); + while (!hasEverConnected) { + checkError(); + if (!running) { + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation stopped before the first successful connection"); + } + if (Thread.interrupted()) { + Thread.currentThread().interrupt(); + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation was interrupted"); + } + if (timeoutNanos - (System.nanoTime() - start) <= 0) { + throw new LineSenderSchemaException( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema negotiation timed out before the first successful connection"); + } + LockSupport.parkNanos(Math.min(50_000L, + timeoutNanos - (System.nanoTime() - start))); + } + // Publish negotiation before hasEverConnected; recovered frames alone + // do not establish the first connection's row policy. + return hasNegotiatedSchema; + } + public boolean isRunning() { return running; } @@ -1584,6 +1699,10 @@ public synchronized void start() { throw new IllegalStateException("already started"); } running = true; + if (engine.requiresSchema() && client != null && !client.isQwpSchemaEnabled()) { + running = false; + throw new QwpSchemaCapabilityMismatchException(); + } // Position the cursor at the first unsent FSN before spinning the // I/O thread. For a fresh sender, ackedFsn=-1 → start at FSN 0, // which lands on the (empty) initial active — same as the prior @@ -1732,6 +1851,11 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM if (!running) { return; } + isWireUp = false; + WebSocketClient failedClient = client; + if (failedClient != null) { + schemaCoordinator.connectionLost(failedClient, shouldAbandonSchemaLookupOnDisconnect); + } if (reconnectFactory == null) { recordFatal(initial); return; @@ -1788,6 +1912,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM attempts++; totalReconnectAttempts.incrementAndGet(); try { + reconnectFactory.setSchemaRequired(engine.requiresSchema()); WebSocketClient newClient = reconnectFactory.reconnect(connectCancellation); if (newClient != null) { if (!running) { @@ -1884,6 +2009,11 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, attempts, e.getMessage()); lastLogNanos = now; } + } catch (QwpSchemaCapabilityMismatchException e) { + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + LOG.warn("schema framing unavailable during {}; retaining backlog and retrying: {}", + phase, e.getMessage()); } catch (QwpDurableAckMismatchException e) { if (endpointPolicyFailureIsTerminal()) { // Orphans hand a capability gap back to BackgroundDrainer's @@ -2376,7 +2506,10 @@ private void ioLoop() { attemptInitialConnect(); } while (running) { - boolean didWork = trySendOne(); + boolean didWork = trySendSchemaRequest(); + if (trySendOne()) { + didWork = true; + } // 1. Try to send next frame(s). // 2. Try to receive ACKs. if (tryReceiveAcks()) { @@ -2430,6 +2563,7 @@ private void ioLoop() { } } } finally { + schemaCoordinator.close(); // Release native-segment lifetime before publishing I/O-thread // completion or running delegated engine cleanup. releaseSendingSegment(); @@ -2549,6 +2683,7 @@ private void positionCursorInSegment(MmapSegment seg, long targetFsn) { * every rethrow delivers the same instance. */ private void recordFatal(Throwable t) { + schemaCoordinator.close(); if (terminalError == null) { terminalError = t instanceof LineSenderException ? (LineSenderException) t @@ -2646,8 +2781,21 @@ private void snapshotReplayTarget() { * {@link #trySendOne} call replays the first unacked frame. */ private void swapClient(WebSocketClient newClient) { + if (engine.requiresSchema() && !newClient.isQwpSchemaEnabled()) { + try { + newClient.close(); + } catch (Throwable ignored) { + // best-effort + } + throw new QwpSchemaCapabilityMismatchException(); + } WebSocketClient old = this.client; this.client = newClient; + // The schema cache survives the reconnect. The server ingests by column name + // and ignores the identity a frame carries, except to decide on feedback. A + // new connection has reported nothing yet, so the first ACK for each table + // refreshes any snapshot that went stale during the outage. Clearing here + // would cost one blocking DESCRIBE per table on the next batch. this.lastReconnectError = null; if (old != null) { try { @@ -2688,6 +2836,10 @@ private void swapClient(WebSocketClient newClient) { // dies inside the dictionary catch-up above was never fully // established, so the sender stays INITIALIZING and a subsequent // endpoint-policy rejection still latches the startup terminal. + if (newClient.isQwpSchemaEnabled()) { + this.hasNegotiatedSchema = true; + } + this.isWireUp = true; this.hasEverConnected = true; } @@ -3390,12 +3542,53 @@ private boolean tryReceiveAcks() { return any; } + private boolean trySendSchemaRequest() { + if (!schemaCoordinator.hasPendingRequest()) { + return false; + } + WebSocketClient activeClient = client; + if (activeClient == null) { + return false; + } + QwpSchemaCoordinator.Request request = schemaCoordinator.requestToSend(activeClient); + if (request == null) { + return false; + } + if (!activeClient.isQwpSchemaEnabled()) { + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "server did not negotiate schema lookup support"); + return true; + } + long remainingNanos = request.remainingNanos(); + if (remainingNanos <= 0) { + schemaCoordinator.fail(request, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + "schema lookup timed out before send"); + return true; + } + int timeoutMillis = (int) Math.min(Integer.MAX_VALUE, + 1L + (remainingNanos - 1L) / 1_000_000L); + try { + activeClient.sendBinary(request.message, timeoutMillis); + return true; + } catch (Throwable t) { + if (t instanceof Error) { + throw (Error) t; + } + fail(t); + return true; + } + } + /** * Returns true if at least one frame was sent (caller skips the park). * Bounded: sends at most one frame per call so the ACK side gets * scheduling fairness. */ private boolean trySendOne() { + if (engine.requiresSchema() && (client == null || !client.isQwpSchemaEnabled())) { + fail(new QwpSchemaCapabilityMismatchException()); + return false; + } if (orphanSkipTipFsn >= 0 && fsnAtZero + nextWireSeq >= orphanSkipStartFsn) { // The send cursor reached the orphaned deferred tail. Its frames // belong to an aborted transaction and must never be transmitted @@ -3485,6 +3678,14 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // The producer latches the requirement before publishing an extended + // frame. Re-check after observing publication so an append racing the + // early guard cannot expose either its legacy prefix or the extended + // frame to an already-connected legacy peer. + if (engine.requiresSchema() && (client == null || !client.isQwpSchemaEnabled())) { + fail(new QwpSchemaCapabilityMismatchException()); + return false; + } // Torn-dictionary guard. sentDictCount is this loop's model of how many ids the // CURRENT server has been told about. A frame whose delta starts ABOVE that // coverage references ids the server was never given, and the server now rejects @@ -3631,6 +3832,15 @@ public enum ReconnectPolicy { public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; + /** + * Tells the factory whether the engine currently holds unacked + * schema-framed frames, so it requests schema framing and rejects peers + * without it. Callers refresh this before each attempt; the send loop + * re-checks the engine when it installs the client. + */ + default void setSchemaRequired(boolean isRequired) { + } + /** * Whether this factory re-derives its {@code Authorization} header from a caller-supplied token * provider on every attempt, rather than presenting a constant captured once. @@ -3862,11 +4072,44 @@ boolean isDurableUnder(CharSequenceLongHashMap watermarks) { private final class ResponseHandler implements WebSocketFrameHandler { @Override public void onBinaryMessage(long payloadPtr, int payloadLen) { - if (!response.readFrom(payloadPtr, payloadLen)) { + if (payloadLen >= Integer.BYTES + && Unsafe.getUnsafe().getInt(payloadPtr) == QwpConstants.MAGIC_MESSAGE) { + WebSocketClient responseClient = client; + final long requestId; + try { + if (responseClient == null || !responseClient.isQwpSchemaEnabled()) { + throw new IllegalArgumentException("schema control response was not negotiated"); + } + requestId = QwpSchemaProtocol.peekResponseRequestId(payloadPtr, payloadLen); + } catch (RuntimeException e) { + schemaCoordinator.invalidResponse(responseClient, 0); + fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); + return; + } + if (!schemaCoordinator.isLiveResponse(responseClient, requestId)) { + return; + } + final QwpSchemaResponse schema; + try { + schema = QwpSchemaProtocol.decodeResponse(payloadPtr, payloadLen); + } catch (RuntimeException e) { + if (!schemaCoordinator.invalidResponse(responseClient, requestId)) { + return; + } + fail(new LineSenderException("Invalid schema control response: " + e.getMessage(), e)); + return; + } + schemaCoordinator.completeResponse(responseClient, schema); + return; + } + if (!response.readFrom(payloadPtr, payloadLen, client != null && client.isQwpSchemaEnabled())) { fail(new LineSenderException( "Invalid ACK response payload [length=" + payloadLen + ']')); return; } + if (response.hasSchemaUpdates() || response.isSchemaInvalidation()) { + schemaCoordinator.applyFeedback(response); + } long wireSeq = response.getSequence(); if (response.isSuccess()) { // Same sanity clamp as legacy: don't trust an ACK beyond diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java new file mode 100644 index 000000000..7829da2d4 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/QwpSchemaCoordinator.java @@ -0,0 +1,338 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.std.Chars; + +import java.util.HashMap; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; +import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; +import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; + +final class QwpSchemaCoordinator { + // Bounded by the tables this sender writes, like its per-table buffers, + // each far heavier than an entry, so it needs no eviction. + private final HashMap cache = new HashMap<>(); + private volatile boolean closed; + private volatile boolean hasPendingRequest; + private long nextRequestId = 1; + private Request request; + + QwpSchemaResponse resolve(CharSequence tableName, long timeoutMillis, boolean refresh) { + final long startNanos = System.nanoTime(); + if (timeoutMillis < 0) { + throw new IllegalArgumentException("timeoutMillis must be non-negative"); + } + if (tableName == null || tableName.length() == 0 + || tableName.length() > QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH) { + throw new IllegalArgumentException("table name must contain 1 to " + + QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + " UTF-16 units"); + } + final String key = normalize(tableName); + final long timeoutNanos = TimeUnit.MILLISECONDS.toNanos(timeoutMillis); + synchronized (this) { + if (closed) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema coordinator is closed"); + } + if (!refresh) { + QwpSchemaResponse cached = cache.get(key); + if (cached != null) { + return cached; + } + } else { + cache.remove(key); + } + if (timeoutNanos - (System.nanoTime() - startNanos) <= 0) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); + } + if (request != null) { + throw failure(SCHEMA_UNAVAILABLE, key, "another schema lookup is already in progress"); + } + Request own = new Request(key, tableName.toString(), startNanos, timeoutNanos); + request = own; + // The I/O loop polls hasPendingRequest on every pass; waking it would + // cut its reconnect backoff short while the wire is down. + hasPendingRequest = true; + while (!own.done) { + try { + await(own, key); + } catch (LineSenderSchemaException e) { + if (request == own) { + request = null; + hasPendingRequest = false; + notifyAll(); + } + throw e; + } + } + if (own.error != null) { + throw own.error; + } + return own.response; + } + } + + synchronized Request requestToSend(WebSocketClient client) { + if (request == null || request.done || request.sentClient != null) { + return null; + } + if (request.remainingNanos() <= 0) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema lookup timed out")); + return null; + } + // Every send attempt, first or after a reconnect, takes a fresh id, so a + // reply to an earlier attempt can never match this one. + if (request.id == 0) { + if (nextRequestId <= 0 || nextRequestId == Long.MAX_VALUE) { + complete(request, null, failure(UNSUPPORTED_FEATURE, request.key, "schema request id space is exhausted")); + return null; + } + request.id = nextRequestId++; + request.message = QwpSchemaProtocol.encodeDescribe(request.id, request.tableName); + } + request.sentClient = client; + return request; + } + + boolean hasPendingRequest() { + return hasPendingRequest; + } + + synchronized boolean isLiveResponse(WebSocketClient client, long requestId) { + Request current = request; + return current != null && !current.done && current.sentClient == client + && current.id == requestId && current.remainingNanos() > 0; + } + + synchronized void completeResponse(WebSocketClient client, QwpSchemaResponse response) { + Request current = request; + if (current == null || current.done || current.sentClient != client + || current.id != response.getRequestId()) { + return; + } + if (current.remainingNanos() <= 0) { + complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema lookup timed out")); + return; + } + if (isCacheable(response.getResult())) { + cache.put(current.key, response); + complete(current, response, null); + return; + } + cache.remove(current.key); + switch (response.getResult()) { + case QwpSchemaProtocol.RESULT_DENIED: + complete(current, null, failure(ACCESS_DENIED, current.key, "schema access denied")); + return; + case QwpSchemaProtocol.RESULT_UNAVAILABLE: + complete(current, null, failure(SCHEMA_UNAVAILABLE, current.key, "schema is unavailable")); + return; + default: + complete(current, null, failure(UNSUPPORTED_FEATURE, current.key, "unsupported schema result")); + } + } + + synchronized void fail(Request own, LineSenderSchemaException.Reason reason, String detail) { + if (request == own && !own.done) { + complete(own, null, failure(reason, own.key, detail)); + } + } + + void applyFeedback(WebSocketResponse feedback) { + if (feedback.isSchemaInvalidation()) { + clearCache(); + } + for (int i = 0, n = feedback.getSchemaUpdateCount(); i < n; i++) { + QwpSchemaResponse schema = feedback.getSchemaUpdate(i); + String key = normalize(feedback.getSchemaUpdateTableName(i)); + synchronized (this) { + if (closed) { + return; + } + if (isCacheable(schema.getResult())) { + cache.put(key, schema); + } else { + cache.remove(key); + } + } + } + } + + /** + * Records that {@code client} died. With {@code abandon} false the producer's + * request keeps its deadline and the replacement connection re-sends it; with + * {@code abandon} true the request fails right away, so a producer that can + * fall back to the legacy contract does not sit out the rest of its budget on + * a wire that is known to be down. + */ + synchronized void connectionLost(WebSocketClient client, boolean abandon) { + if (request == null || request.done) { + return; + } + if (abandon) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "connection lost during schema lookup")); + return; + } + if (request.sentClient == client) { + // Keep the producer's request and deadline; only its wire attempt ended. + // A fresh ID also rejects old replies delivered on the replacement connection. + request.sentClient = null; + request.id = 0; + } + } + + /** + * Cache-only lookup: the schema this coordinator already holds for the table, + * or {@code null}. Never sends a request and never waits. + */ + synchronized QwpSchemaResponse peek(CharSequence tableName) { + if (closed) { + return null; + } + return cache.get(normalize(tableName)); + } + + synchronized boolean invalidResponse(WebSocketClient client, long requestId) { + // Zero means the malformed header could not identify a request. For an + // identifiable reply, cancellation must share the liveness check's lock. + if (requestId != 0 && !isLiveResponse(client, requestId)) { + return false; + } + if (request != null && (request.sentClient == null || request.sentClient == client)) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "invalid schema control response")); + } + return true; + } + + synchronized void clearCache() { + if (!closed) { + cache.clear(); + } + } + + /** + * Drops the cached KNOWN snapshot of {@code tableName} when it still has the + * given identity. A newer snapshot, such as one that ACK feedback installed + * after the caller pinned its binding, stays cached. + */ + synchronized void evictKnown(CharSequence tableName, int tableId, long metadataVersion) { + if (closed) { + return; + } + String key = normalize(tableName); + QwpSchemaResponse cached = cache.get(key); + if (cached != null + && cached.getResult() == QwpSchemaProtocol.RESULT_KNOWN + && cached.getTableId() == tableId + && cached.getMetadataVersion() == metadataVersion) { + cache.remove(key); + } + } + + synchronized void close() { + closed = true; + cache.clear(); + if (request != null) { + complete(request, null, failure(SCHEMA_UNAVAILABLE, request.key, "schema coordinator is closed")); + } + notifyAll(); + } + + /** + * Results that stay true until the table's schema version changes, which + * the server then reports through ACK feedback. TOO_LARGE is a negative entry, + * like MISSING: without it, AUTO would pay a DESCRIBE round trip every batch + * for a table too wide to describe. DENIED and UNAVAILABLE may clear at any + * time, so they are never cached. + */ + private static boolean isCacheable(int result) { + return result == QwpSchemaProtocol.RESULT_KNOWN + || result == QwpSchemaProtocol.RESULT_MISSING + || result == QwpSchemaProtocol.RESULT_TOO_LARGE; + } + + private static LineSenderSchemaException failure(LineSenderSchemaException.Reason reason, String table, String detail) { + return new LineSenderSchemaException(reason, "schema lookup failed [reason=" + reason + + ", table=" + table + ", detail=" + detail + ']'); + } + + /** + * Lower-cases a table name into a cache key. A String that is already lower + * case is its own key, so the sender's pre-computed + * {@code QwpTableBuffer.getSchemaKey()} costs no allocation on a cache hit. + */ + private static String normalize(CharSequence tableName) { + if (tableName instanceof String) { + String name = (String) tableName; + for (int i = 0, n = name.length(); i < n; i++) { + char c = name.charAt(i); + if (Character.toLowerCase(c) != c) { + return Chars.toLowerCase(name); + } + } + return name; + } + return Chars.toLowerCase(tableName); + } + + private void await(Request request, String key) { + long remaining = request.remainingNanos(); + if (remaining <= 0) { + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup timed out"); + } + try { + long millis = remaining / 1_000_000L; + int nanos = (int) (remaining % 1_000_000L); + wait(millis, nanos); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw failure(SCHEMA_UNAVAILABLE, key, "schema lookup was interrupted"); + } + } + + private void complete(Request own, QwpSchemaResponse response, LineSenderSchemaException error) { + own.response = response; + own.error = error; + own.done = true; + if (request == own) { + request = null; + hasPendingRequest = false; + } + notifyAll(); + } + + + static final class Request { + final long startNanos; + final long timeoutNanos; + long id; + final String key; + final String tableName; + byte[] message; + boolean done; + LineSenderSchemaException error; + QwpSchemaResponse response; + WebSocketClient sentClient; + + Request(String key, String tableName, long startNanos, long timeoutNanos) { + this.key = key; + this.tableName = tableName; + this.startNanos = startNanos; + this.timeoutNanos = timeoutNanos; + } + + long remainingNanos() { + return timeoutNanos - (System.nanoTime() - startNanos); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index 6a7f70df3..9fdd0e2ba 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -65,6 +65,8 @@ final class RecoveredFrameAnalysis implements QuietCloseable { private int runningRawCount; private int runningRawLen; private long runningCoverage; + // FSN of the newest FLAG_SCHEMA frame visited; -1 when none. + private long maxSchemaFsn = -1L; RecoveredFrameAnalysis(int baseline, long ackedFsn) { this.baseline = baseline; @@ -81,6 +83,11 @@ void accept(long fsn, long payload, int payloadLen) { byte flags = isQwp ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS) : 0; + if (isQwp && (flags & QwpConstants.FLAG_SCHEMA) != 0) { + // Includes a retained deferred tail: it may still be replay-owned + // until the engine's orphan-tail retirement acknowledges it. + maxSchemaFsn = fsn; + } if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) { foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen); } @@ -189,6 +196,10 @@ long maxDeltaStart() { return committedMaxDeltaStart; } + long maxSchemaFsn() { + return maxSchemaFsn; + } + long rawAddr() { return rawAddr; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java index 074cf0e3f..77f371301 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpConstants.java @@ -50,6 +50,8 @@ public final class QwpConstants { * Flag bit: Gorilla timestamp encoding enabled. */ public static final byte FLAG_GORILLA = 0x04; + /** Flag bit: table headers carry schema identity information. */ + public static final byte FLAG_SCHEMA = 0x40; /** * Flag bit: payload region after the prelude is zstd-compressed. Set only * when the handshake negotiated zstd compression. Mirror of the server-side @@ -246,6 +248,7 @@ public static int getFixedTypeSize(byte typeCode) { case TYPE_CHAR: return 2; case TYPE_INT: + case TYPE_IPv4: case TYPE_FLOAT: return 4; case TYPE_LONG: @@ -292,6 +295,9 @@ public static String getTypeName(byte typeCode) { case TYPE_INT: name = "INT"; break; + case TYPE_IPv4: + name = "IPv4"; + break; case TYPE_LONG: name = "LONG"; break; @@ -359,6 +365,7 @@ public static boolean isFixedWidthType(byte typeCode) { (int) typeCode == TYPE_SHORT || (int) typeCode == TYPE_CHAR || (int) typeCode == TYPE_INT || + (int) typeCode == TYPE_IPv4 || (int) typeCode == TYPE_LONG || (int) typeCode == TYPE_FLOAT || (int) typeCode == TYPE_DOUBLE || diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java new file mode 100644 index 000000000..d46c10302 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java @@ -0,0 +1,1965 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cairo.MicrosTimestampDriver; +import io.questdb.client.cairo.NanosTimestampDriver; +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.std.Decimals; +import io.questdb.client.std.NumericException; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.bytes.DirectByteSlice; +import io.questdb.client.std.fastdouble.FastDoubleParser; +import io.questdb.client.std.fastdouble.FastFloatParser; +import io.questdb.client.std.str.StringSink; +import io.questdb.client.std.str.Utf8StringSink; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.LineSenderSchemaException.Reason.ACCESS_DENIED; +import static io.questdb.client.LineSenderSchemaException.Reason.INVALID_VALUE; +import static io.questdb.client.LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE; +import static io.questdb.client.LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE; + +/** + * Converts supported input values against one immutable server schema result + * and appends them to a caller-owned {@link QwpTableBuffer}. A confirmed + * missing table, or a column absent from a known schema, is inferred from the + * first effective supported input value. + * + *

This binding does not own the buffer or its row lifecycle and is not + * thread-safe. The caller completes rows, encodes data and closes the buffer. + * After a setter failure, the caller must invoke both + * {@link QwpTableBuffer#cancelCurrentRow()} and + * {@link QwpTableBuffer#rollbackUncommittedColumns()} before continuing or + * completing another row. {@link QwpTableBuffer#reset()} preserves the binding; + * {@link QwpTableBuffer#clear()} invalidates it.

+ * + *

The caller must encode the buffer with the matching symbol-dictionary + * family: local dictionaries for ownerless buffers, or the owning sender's + * corresponding global dictionary for sender-backed buffers.

+ */ +public final class QwpSchemaBinding { + private static final int FIRST_DASH = 8; + private static final int SECOND_DASH = 13; + private static final int THIRD_DASH = 18; + private static final int FOURTH_DASH = 23; + private static final int UUID_LENGTH = 36; + + private final QwpTableBuffer buffer; + private final QwpSchemaResponse schema; + private final String tableName; + private boolean hasTargetTypeConflict; + private Decimal256 decimalTextScratch; + private StringSink charTextSink; + private StringSink floatingTextSink; + private StringSink numericTextSink; + private Utf8StringSink timestampTextSink; + private StringSink uuidTextSink; + + /** + * Attaches {@code schema} to an empty, unbound caller-owned buffer. The + * caller must correlate a + * positive request id with its DESCRIBE table, or obtain request-id-zero + * schema from the matching named write-feedback entry. + */ + public QwpSchemaBinding(QwpTableBuffer buffer, QwpSchemaResponse schema) { + if (buffer == null) { + throw new NullPointerException("buffer"); + } + String tableName = buffer.getTableName(); + if (tableName == null || !TableUtils.isValidTableName(tableName, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { + throw new IllegalArgumentException("invalid table name"); + } + if (schema == null) { + throw new NullPointerException("schema"); + } + this.tableName = tableName; + this.schema = schema; + requireUsableSchema(); + this.buffer = buffer; + buffer.attachSchemaBinding(this); + } + + /** + * Appends an owned copy of a BINARY value to an exact BINARY target. + * The caller retains the class-level row-cancellation and uncommitted-column rollback + * responsibility on failure. + */ + public QwpSchemaBinding binaryColumn(CharSequence name, byte[] value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + if (value == null) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary value cannot be null"); + } + column.addBinary(value); + return this; + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal64 value) { + return decimalColumn(name, value, "DECIMAL64", QwpConstants.TYPE_DECIMAL64); + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal128 value) { + return decimalColumn(name, value, "DECIMAL128", QwpConstants.TYPE_DECIMAL128); + } + + public QwpSchemaBinding decimalColumn(CharSequence name, Decimal256 value) { + return decimalColumn(name, value, "DECIMAL256", QwpConstants.TYPE_DECIMAL256); + } + + /** + * Parses the public textual-decimal overload after target selection and + * duplicate suppression, then applies the same DECIMAL256 conversion rules + * as the typed overload. The caller owns {@code scratch}; this binding does + * not retain it. + */ + public QwpSchemaBinding decimalColumn(CharSequence name, CharSequence value, Decimal256 scratch) { + buffer.requireSchemaBinding(this); + if (value == null || value.length() == 0) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DECIMAL256"); + int targetType = targetType(index, ColumnType.DECIMAL256); + if (index < 0) { + if (!isInferredWritable(existing, name, "DECIMAL256", QwpConstants.TYPE_DECIMAL256)) { + return this; + } + parseDecimalText(name, value, scratch, targetType); + // Keep the legacy public-overload contract: infer DECIMAL256 at + // the parsed natural scale. A parsed special is an effective + // null but does not pin the scale of a later finite value. + claimInferred(existing, name, QwpConstants.TYPE_DECIMAL256).addDecimal256(scratch); + return this; + } + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "DECIMAL256", index, targetType); + if (column == null) { + return this; + } + if (!ColumnType.isDecimal(targetType) + && targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "DECIMAL256", targetType, "conversion is not implemented"); + } + parseDecimalText(name, value, scratch, targetType); + if (scratch.isNull()) { + if (ColumnType.isDecimal(targetType)) { + column.addSchemaDecimalNull(ColumnType.getDecimalScale(targetType)); + } else { + column.addNull(); + } + return this; + } + return appendDecimal(column, name, scratch, "DECIMAL256", targetType); + } + + /** Appends an owned copy of the bytes addressed by {@code slice}. */ + public QwpSchemaBinding binaryColumn(CharSequence name, DirectByteSlice slice) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + if (slice == null) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary slice cannot be null"); + } + appendBinary(name, column, slice.ptr(), slice.size()); + return this; + } + + /** + * Appends an owned copy of {@code [ptr, ptr + len)}. The source memory need only remain + * valid for the duration of this call. + */ + public QwpSchemaBinding binaryColumn(CharSequence name, long ptr, long len) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer column = targetColumn(name, "BINARY", ColumnType.BINARY); + if (column == null) { + return this; + } + appendBinary(name, column, ptr, len); + return this; + } + + public QwpSchemaBinding boolColumn(CharSequence name, boolean value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "BOOLEAN"); + int targetType = targetType(index, ColumnType.BOOLEAN); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "BOOLEAN", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.BOOLEAN: + column.addBoolean(value); + break; + case ColumnType.BYTE: + column.addByte((byte) (value ? 1 : 0)); + break; + case ColumnType.SHORT: + column.addShort((short) (value ? 1 : 0)); + break; + case ColumnType.INT: + column.addInt(value ? 1 : 0); + break; + case ColumnType.LONG: + column.addLong(value ? 1 : 0); + break; + case ColumnType.FLOAT: + column.addFloat(value ? 1 : 0); + break; + case ColumnType.DOUBLE: + column.addDouble(value ? 1 : 0); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value ? "true" : "false"); + break; + default: + throw targetTypeConflict(name, "BOOLEAN", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 1); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[][] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 2); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, double[][][] values) { + buffer.requireSchemaBinding(this); + if (values == null) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(name, 3); + if (column != null) { + column.addDoubleArray(values); + } + return this; + } + + public QwpSchemaBinding doubleArray(CharSequence name, DoubleArray array) { + buffer.requireSchemaBinding(this); + if (array == null) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DOUBLE_ARRAY"); + if (existing != null && existing.getSize() > buffer.getRowCount()) { + return this; + } + QwpTableBuffer.ColumnBuffer column = targetDoubleArrayColumn(existing, name, index, array.getDimensionality()); + if (column != null) { + column.addDoubleArray(array); + } + return this; + } + + public QwpSchemaBinding byteColumn(CharSequence name, byte value) { + return integerColumn(name, value, "BYTE", ColumnType.BYTE, false); + } + + public QwpSchemaBinding doubleColumn(CharSequence name, double value) { + return floatingNumericColumn(name, value, "DOUBLE"); + } + + /** Appends and pins the row's designated timestamp using the schema target precision. */ + public QwpSchemaBinding designatedTimestamp(long value, ChronoUnit unit) { + buffer.requireSchemaBinding(this); + int targetType = schema.getResult() == QwpSchemaProtocol.RESULT_MISSING + ? (unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO) + : schema.getColumnType(requireDesignatedTimestamp("TIMESTAMP")); + if (unit == null) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "timestamp unit is null"); + } + if (!isSupportedTimestampUnit(unit)) { + throw unsupported(null, "TIMESTAMP", targetType, "timestamp unit is not supported: " + unit); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_MICRO + ? MicrosTimestampDriver.INSTANCE.from(value, unit) + : NanosTimestampDriver.INSTANCE.from(value, unit); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + designatedColumn(targetType, "TIMESTAMP").addLong(converted); + return this; + } + + /** Appends and pins an Instant as the row's designated timestamp. */ + public QwpSchemaBinding designatedTimestamp(Instant value) { + buffer.requireSchemaBinding(this); + int targetType = schema.getResult() == QwpSchemaProtocol.RESULT_MISSING + ? ColumnType.TIMESTAMP_MICRO + : schema.getColumnType(requireDesignatedTimestamp("TIMESTAMP")); + if (value == null) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "timestamp value is null"); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_MICRO + ? instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L) + : instantTimestamp(value, 1_000_000_000L, value.getNano()); + } catch (ArithmeticException e) { + throw error(INVALID_VALUE, null, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + designatedColumn(targetType, "TIMESTAMP").addLong(converted); + return this; + } + + private QwpTableBuffer.ColumnBuffer designatedColumn(int targetType, String inputType) { + byte wireType = timestampWireType(targetType); + if (schema.getResult() == QwpSchemaProtocol.RESULT_MISSING) { + QwpTableBuffer.ColumnBuffer inferred = buffer.getExistingColumnByName(""); + if (inferred != null && inferred.getType() != wireType) { + throw unsupported(null, inputType, -1, "inferred designated timestamp type conflict [inferredType=" + + QwpConstants.getTypeName(inferred.getType()) + ']'); + } + } + return buffer.getOrCreateDesignatedTimestampColumn(wireType); + } + + public QwpSchemaBinding floatColumn(CharSequence name, float value) { + return floatingNumericColumn(name, value, "FLOAT"); + } + + public QwpSchemaBinding geoHashColumn(CharSequence name, long bits, int precisionBits) { + buffer.requireSchemaBinding(this); + if (precisionBits < 1 || precisionBits > 60) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "invalid GEOHASH precision: " + precisionBits + " (must be 1-60)"); + } + int sourceType = ColumnType.getGeoHashTypeWithBits(precisionBits); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "GEOHASH"); + int targetType = targetType(index, sourceType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "GEOHASH", index, targetType); + if (column == null) { + return this; + } + long value = bits & ((1L << precisionBits) - 1L); + if (!ColumnType.isArray(targetType) && ColumnType.isGeoHash(targetType)) { + int targetBits = ColumnType.getGeoHashBits(targetType); + if (targetBits != precisionBits) { + throw unsupported(name, "GEOHASH", targetType, + "GEOHASH precision mismatch [sourceBits=" + precisionBits + ", targetBits=" + targetBits + ']'); + } + column.addGeoHash(value, precisionBits); + } else if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + column.addString(formatGeoHash(value, precisionBits)); + } else { + throw targetTypeConflict(name, "GEOHASH", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding geoHashColumn(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + if (value == null) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "GEOHASH string cannot be null; omit the setter to write NULL"); + } + int length = value.length(); + if (length == 0) { + throw error(INVALID_VALUE, name, "GEOHASH", null, "GEOHASH string cannot be empty"); + } + if (length > 12) { + throw error(INVALID_VALUE, name, "GEOHASH", null, + "GEOHASH string exceeds 12 characters: " + length); + } + final long bits; + try { + bits = Numbers.parseGeoHashBase32(value, 0, length); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "GEOHASH", null, "invalid GEOHASH string"); + } + return geoHashColumn(name, bits, length * 5); + } + + public QwpSchemaBinding intColumn(CharSequence name, int value) { + return integerColumn(name, value, "INT", ColumnType.INT, value == Integer.MIN_VALUE); + } + + public QwpSchemaBinding ipv4Column(CharSequence name, int address) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "IPv4"); + int targetType = targetType(index, ColumnType.IPv4); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "IPv4", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.IPv4 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "IPv4", targetType, "conversion is not implemented"); + } + if (address == Numbers.IPv4_NULL) { + column.addNull(); + } else if (targetType == ColumnType.IPv4) { + column.addIPv4(address); + } else { + column.addString(formatIPv4(address)); + } + return this; + } + + public QwpSchemaBinding ipv4Column(CharSequence name, CharSequence address) { + buffer.requireSchemaBinding(this); + if (address == null) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "IPv4"); + int targetType = targetType(index, ColumnType.IPv4); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "IPv4", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.IPv4 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "IPv4", targetType, "conversion is not implemented"); + } + if (Chars.equalsIgnoreCase("null", address) || Chars.equals("0.0.0.0", address)) { + throw error(INVALID_VALUE, name, "IPv4", targetType, + "NULL sentinel inputs are rejected; pass a null reference or omit the setter"); + } + final int packed; + try { + packed = Numbers.parseIPv4(address); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "IPv4", targetType, "invalid IPv4 address"); + } + if (packed == Numbers.IPv4_NULL) { + column.addNull(); + } else if (targetType == ColumnType.IPv4) { + column.addIPv4(packed); + } else { + column.addString(formatIPv4(packed)); + } + return this; + } + + public QwpSchemaBinding longColumn(CharSequence name, long value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "LONG"); + int targetType = targetType(index, ColumnType.LONG); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "LONG", index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType) + && !isTextTarget(targetType) + && !ColumnType.isDecimal(targetType) + && targetType != ColumnType.DATE + && targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO) { + throw targetTypeConflict(name, "LONG", targetType, "conversion is not implemented"); + } + if (ColumnType.isDecimal(targetType)) { + appendIntegerDecimal(column, name, value, "LONG", targetType, value == Long.MIN_VALUE); + return this; + } + if (value == Long.MIN_VALUE) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw invalidRange(name, "LONG", targetType); + } + column.addByte((byte) value); + break; + case ColumnType.SHORT: + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw invalidRange(name, "LONG", targetType); + } + column.addShort((short) value); + break; + case ColumnType.INT: + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw invalidRange(name, "LONG", targetType); + } + column.addInt((int) value); + break; + case ColumnType.LONG: + case ColumnType.DATE: + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + column.addLong(value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble((double) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatLong(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatLong(value)); + break; + default: + throw targetTypeConflict(name, "LONG", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding long256Column(CharSequence name, long l0, long l1, long l2, long l3) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "LONG256"); + int targetType = targetType(index, ColumnType.LONG256); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "LONG256", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.LONG256 + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "LONG256", targetType, "conversion is not implemented"); + } + if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE + && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { + column.addNull(); + } else if (targetType == ColumnType.LONG256) { + column.addLong256(l0, l1, l2, l3); + } else { + column.addString(formatLong256(l0, l1, l2, l3)); + } + return this; + } + + public QwpSchemaBinding shortColumn(CharSequence name, short value) { + return integerColumn(name, value, "SHORT", ColumnType.SHORT, false); + } + + public QwpSchemaBinding stringColumn(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "STRING"); + int targetType = targetType(index, ColumnType.VARCHAR); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "STRING", index, targetType); + if (column == null) { + return this; + } + if (!ColumnType.isArray(targetType) && ColumnType.isGeoHash(targetType)) { + int precision = ColumnType.getGeoHashBits(targetType); + column.initGeoHashPrecision(precision); + if (value == null || value.length() == 0) { + column.addNull(); + } else { + appendStringGeoHash(column, name, value, precision, targetType); + } + return this; + } + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (value == null) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaStringDecimal(value, targetPrecision, targetScale)) { + throw targetValueConflict(name, "STRING", targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid decimal text"); + } + return this; + } + if (value == null) { + column.addNull(); + } else { + switch (targetType) { + case ColumnType.BYTE: + case ColumnType.SHORT: + case ColumnType.INT: + case ColumnType.LONG: + case ColumnType.FLOAT: + case ColumnType.DOUBLE: + appendStringNumeric(column, name, value, targetType); + break; + case ColumnType.BOOLEAN: + if (value.length() == 1 && value.charAt(0) == '1') { + column.addBoolean(true); + } else if (value.length() == 1 && value.charAt(0) == '0') { + column.addBoolean(false); + } else if (Chars.equalsLowerCaseAscii(value, "true")) { + column.addBoolean(true); + } else if (Chars.equalsLowerCaseAscii(value, "false")) { + column.addBoolean(false); + } else { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid BOOLEAN text"); + } + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value); + break; + case ColumnType.SYMBOL: + column.addSymbol(value); + break; + case ColumnType.BINARY: + column.addString(value); + break; + case ColumnType.CHAR: + column.addShort((short) stringToChar(value)); + break; + case ColumnType.DATE: + appendStringDate(column, name, value, targetType); + break; + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + appendStringTimestamp(column, name, value, targetType); + break; + case ColumnType.UUID: + long lo; + long hi; + try { + checkUuid(value); + lo = parseUuidLo(value); + hi = parseUuidHi(value); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid UUID text"); + } + column.addUuid(hi, lo); + break; + case ColumnType.LONG256: + appendStringLong256(column, name, value); + break; + default: + throw targetTypeConflict(name, "STRING", targetType, "conversion is not implemented"); + } + } + return this; + } + + public QwpSchemaBinding charColumn(CharSequence name, char value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "CHAR"); + int targetType = targetType(index, ColumnType.CHAR); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "CHAR", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.CHAR: + column.addShort((short) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + if (Character.isSurrogate(value)) { + throw error(INVALID_VALUE, name, "CHAR", targetType, "surrogate cannot be converted to text"); + } + StringSink sink = charTextSink; + if (sink == null) { + charTextSink = sink = new StringSink(1); + } else { + sink.clear(); + } + sink.put(value); + column.addString(sink); + break; + default: + throw targetTypeConflict(name, "CHAR", targetType, "conversion is not implemented"); + } + return this; + } + + public QwpSchemaBinding symbol(CharSequence name, CharSequence value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "SYMBOL"); + int targetType = targetType(index, ColumnType.SYMBOL); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "SYMBOL", index, targetType); + if (column == null) { + return this; + } + switch (targetType) { + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(value); + break; + case ColumnType.SYMBOL: + column.addSymbol(value); + break; + default: + throw targetTypeConflict(name, "SYMBOL", targetType, "conversion is not implemented"); + } + return this; + } + + /** + * Appends a non-designated timestamp supplied in an explicitly supported fixed-duration unit + * to an exact TIMESTAMP, TIMESTAMP_NS, STRING or VARCHAR schema target. Text targets normalize + * through checked microseconds and use the server's UTC millisecond presentation. Conversion + * fails on overflow; nanos converted to micros truncate toward zero. The caller retains the + * class-level row-cancellation and uncommitted-column rollback responsibility on failure. + */ + public QwpSchemaBinding timestampColumn(CharSequence name, long value, ChronoUnit unit) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "TIMESTAMP"); + int targetType = targetType(index, + unit == ChronoUnit.NANOS ? ColumnType.TIMESTAMP_NANO : ColumnType.TIMESTAMP_MICRO); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "TIMESTAMP", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "TIMESTAMP", targetType, "conversion is not implemented"); + } + if (unit == null) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp unit is null"); + } + if (!isSupportedTimestampUnit(unit)) { + throw unsupported(name, "TIMESTAMP", targetType, "timestamp unit is not supported: " + unit); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_NANO + ? NanosTimestampDriver.INSTANCE.from(value, unit) + : MicrosTimestampDriver.INSTANCE.from(value, unit); + } catch (ArithmeticException e) { + throw targetValueConflict(name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + appendTimestampText(column, converted); + } else { + column.addLong(converted); + } + return this; + } + + /** + * Appends a non-designated Instant to an exact TIMESTAMP, TIMESTAMP_NS, STRING or VARCHAR + * schema target. Timestamp targets preserve their precision; text targets normalize through + * checked microseconds and use the server's UTC millisecond presentation. Final values outside + * the selected signed-long range are rejected. The caller retains the class-level rollback + * responsibility on failure. + */ + public QwpSchemaBinding timestampColumn(CharSequence name, Instant value) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "TIMESTAMP"); + int targetType = targetType(index, ColumnType.TIMESTAMP_MICRO); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "TIMESTAMP", index, targetType); + if (column == null) { + return this; + } + if (targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO + && targetType != ColumnType.STRING + && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "TIMESTAMP", targetType, "conversion is not implemented"); + } + if (value == null) { + throw error(INVALID_VALUE, name, "TIMESTAMP", targetType, "timestamp value is null"); + } + final long converted; + try { + converted = targetType == ColumnType.TIMESTAMP_NANO + ? instantTimestamp(value, 1_000_000_000L, value.getNano()) + : instantTimestamp(value, 1_000_000L, value.getNano() / 1_000L); + } catch (ArithmeticException e) { + throw targetValueConflict(name, "TIMESTAMP", targetType, "value is outside target timestamp range"); + } + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + appendTimestampText(column, converted); + } else { + column.addLong(converted); + } + return this; + } + + public QwpSchemaBinding uuidColumn(CharSequence name, long lo, long hi) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "UUID"); + int targetType = targetType(index, ColumnType.UUID); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "UUID", index, targetType); + if (column == null) { + return this; + } + if (targetType == ColumnType.UUID) { + column.addUuid(hi, lo); + return this; + } + if (targetType != ColumnType.STRING && targetType != ColumnType.VARCHAR) { + throw targetTypeConflict(name, "UUID", targetType, "conversion is not implemented"); + } + if (lo == Long.MIN_VALUE && hi == Long.MIN_VALUE) { + column.addNull(); + } else { + column.addString(formatUuid(lo, hi)); + } + return this; + } + + /** Rejects an effective input family whose schema-mode conversion is not implemented. */ + public QwpSchemaBinding unsupportedColumn(CharSequence name, String inputType) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + QwpTableBuffer.ColumnBuffer inferred = index < 0 ? existing : null; + if (inferred != null && inferred.getSize() > buffer.getRowCount()) { + return this; + } + if (index < 0) { + throw unsupported(name, inputType, -1, "input conversion is not implemented"); + } + int targetType = schema.getColumnType(index); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column != null) { + throw unsupported(name, inputType, targetType, "input conversion is not implemented"); + } + return this; + } + + private static void checkUuid(CharSequence value) throws NumericException { + if (value.length() != UUID_LENGTH + || value.charAt(FIRST_DASH) != '-' + || value.charAt(SECOND_DASH) != '-' + || value.charAt(THIRD_DASH) != '-' + || value.charAt(FOURTH_DASH) != '-') { + throw NumericException.instance(); + } + } + + private static long parseUuidHi(CharSequence value) throws NumericException { + return (Numbers.parseHexLong(value, 0, FIRST_DASH) << 32) + | (Numbers.parseHexLong(value, FIRST_DASH + 1, SECOND_DASH) << 16) + | Numbers.parseHexLong(value, SECOND_DASH + 1, THIRD_DASH); + } + + private static long parseUuidLo(CharSequence value) throws NumericException { + return (Numbers.parseHexLong(value, THIRD_DASH + 1, FOURTH_DASH) << 48) + | Numbers.parseHexLong(value, FOURTH_DASH + 1, UUID_LENGTH); + } + + public int getTableId() { + return schema.getTableId(); + } + + public long getMetadataVersion() { + return schema.getMetadataVersion(); + } + + /** + * Clears the conflict that {@link #hasTargetTypeConflict()} reports, once the + * caller has acted on it. + */ + public void clearTargetTypeConflict() { + hasTargetTypeConflict = false; + } + + /** + * Reports whether the last rejected value targeted a schema column whose + * snapshot type cannot take it. A column type change after the snapshot + * causes exactly this, so the caller evicts the snapshot from its cache. + * Input families that no column type can take never set it. + */ + public boolean hasTargetTypeConflict() { + return hasTargetTypeConflict; + } + + private LineSenderSchemaException error( + LineSenderSchemaException.Reason reason, + CharSequence column, + CharSequence inputType, + Integer targetType, + String detail + ) { + StringSink message = new StringSink(); + message.put("schema row error [reason=").put(reason.name()).put(", table=").putAsPrintable(tableName); + if (column != null) { + message.put(", column=").putAsPrintable(column); + } + if (inputType != null) { + message.put(", inputType=").put(inputType); + } + if (targetType != null) { + message.put(", targetType=").put(ColumnType.nameOf(targetType)).put('(').put(targetType).put(')'); + } + message.put(", detail=").put(detail).put(']'); + return new LineSenderSchemaException(reason, message); + } + + // Only a column the snapshot knows records the conflict; an inferred column's + // type comes from this batch, so a fresh snapshot could not change the outcome. + private void recordTargetTypeConflict(CharSequence name) { + if (name != null && schema.getColumnIndex(name) >= 0) { + hasTargetTypeConflict = true; + } + } + + private void requireUsableSchema() { + switch (schema.getResult()) { + case QwpSchemaProtocol.RESULT_KNOWN: + case QwpSchemaProtocol.RESULT_MISSING: + return; + case QwpSchemaProtocol.RESULT_DENIED: + throw error(ACCESS_DENIED, null, null, null, "schema access denied"); + case QwpSchemaProtocol.RESULT_UNAVAILABLE: + throw error(SCHEMA_UNAVAILABLE, null, null, null, "schema is temporarily unavailable"); + case QwpSchemaProtocol.RESULT_TOO_LARGE: + throw error(UNSUPPORTED_FEATURE, null, null, null, "schema could not be returned within the discovery response limit"); + default: + throw error(UNSUPPORTED_FEATURE, null, null, null, "unknown schema result"); + } + } + + /** Claims or creates the column for a name the schema lacks; call after {@link #isInferredWritable}. */ + private QwpTableBuffer.ColumnBuffer claimInferred( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + byte wireType + ) { + QwpTableBuffer.ColumnBuffer column = existing != null + ? buffer.claimColumn(existing) + : buffer.getOrCreateColumn(name, wireType, true); + column.setSchemaIndex(-1); + return column; + } + + /** + * Finds the buffer column for {@code name}, advancing the buffer's + * sequential cursor. An invalid name never matches, so the caller still + * rejects it through {@link #targetIndex}. + */ + private QwpTableBuffer.ColumnBuffer existingColumn(CharSequence name) { + return name == null || name.length() == 0 ? null : buffer.findColumn(name); + } + + /** + * Returns false for a duplicate write to a column the schema lacks, and + * rejects a write whose wire type conflicts with the inferred column. + */ + private boolean isInferredWritable( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + String inputType, + byte wireType + ) { + if (existing == null) { + return true; + } + if (existing.getSize() > buffer.getRowCount()) { + return false; + } + if (existing.getType() != wireType) { + throw unsupported(name, inputType, -1, "inferred column type conflict [inferredType=" + + QwpConstants.getTypeName(existing.getType()) + ']'); + } + return true; + } + + private QwpTableBuffer.ColumnBuffer targetColumn(CharSequence name, String inputType, int expectedTargetType) { + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + int targetType = targetType(index, expectedTargetType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column != null && targetType != expectedTargetType) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + return column; + } + + private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn(CharSequence name, int sourceDimensions) { + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, "DOUBLE_ARRAY"); + return targetDoubleArrayColumn(existing, name, index, sourceDimensions); + } + + private QwpTableBuffer.ColumnBuffer targetDoubleArrayColumn( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + int index, + int sourceDimensions + ) { + int targetType = targetType(index, ColumnType.encodeArrayType(ColumnType.DOUBLE, sourceDimensions)); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, "DOUBLE_ARRAY", index, targetType); + if (column == null) { + return null; + } + if (!ColumnType.isArray(targetType) + || ColumnType.decodeArrayElementType(targetType) != ColumnType.DOUBLE) { + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "conversion is not implemented"); + } + int targetDimensions = ColumnType.decodeWeakArrayDimensionality(targetType); + if (targetDimensions < 1 + || targetType != ColumnType.encodeArrayType(ColumnType.DOUBLE, targetDimensions)) { + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, "invalid target array type"); + } + if (targetDimensions != sourceDimensions) { + throw targetTypeConflict(name, "DOUBLE_ARRAY", targetType, + "array dimensionality mismatch [sourceDims=" + sourceDimensions + + ", targetDims=" + targetDimensions + ']'); + } + if (index < 0) { + int inferredDimensions = column.pinInferredArrayDimensionality(sourceDimensions); + if (inferredDimensions != sourceDimensions) { + throw unsupported(name, "DOUBLE_ARRAY", targetType, + "array dimensionality mismatch [sourceDims=" + sourceDimensions + + ", inferredDims=" + inferredDimensions + ']'); + } + } + return column; + } + + /** + * Returns the column {@code name} writes to for the in-progress row, or + * {@code null} for a duplicate write. {@code existing} is the buffer's + * column for {@code name}, found by {@link #existingColumn}. + */ + private QwpTableBuffer.ColumnBuffer targetColumn( + QwpTableBuffer.ColumnBuffer existing, + CharSequence name, + String inputType, + int index, + int targetType + ) { + if (index < 0) { + final byte wireType = wireType(targetType); + if (!isInferredWritable(existing, name, inputType, wireType)) { + return null; + } + if (wireType == 0) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + return claimInferred(existing, name, wireType); + } + if (existing != null && existing.isSchemaResolved()) { + // Every check below depends only on the schema column, and passed + // when this binding created the column. + return buffer.claimColumn(existing); + } + if (index == schema.getDesignatedIndex()) { + throw unsupported(name, inputType, targetType, "designated timestamp writes are not implemented"); + } + final byte wireType = wireType(targetType); + if (wireType == 0) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + if (schema.hasColumnExtensionParameters(index)) { + throw unsupported(name, inputType, targetType, "parameterized target type"); + } + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn(name, wireType, true); + if (column != null) { + column.setSchemaIndex(index); + } + return column; + } + + private byte wireType(int targetType) { + if (ColumnType.isArray(targetType)) { + int dimensions = ColumnType.decodeWeakArrayDimensionality(targetType); + return dimensions >= 1 + && ColumnType.decodeArrayElementType(targetType) == ColumnType.DOUBLE + && targetType == ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions) + ? QwpConstants.TYPE_DOUBLE_ARRAY + : 0; + } + if (ColumnType.isGeoHash(targetType)) { + int bits = ColumnType.getGeoHashBits(targetType); + return bits >= 1 && bits <= 60 && targetType == ColumnType.getGeoHashTypeWithBits(bits) + ? QwpConstants.TYPE_GEOHASH + : 0; + } + if (ColumnType.isDecimal(targetType)) { + int precision = ColumnType.getDecimalPrecision(targetType); + int scale = ColumnType.getDecimalScale(targetType); + if (precision < 1 || precision > Decimals.MAX_PRECISION || scale > precision + || targetType != ColumnType.getDecimalType(precision, scale)) { + return 0; + } + switch (ColumnType.tagOf(targetType)) { + case ColumnType.DECIMAL8: + case ColumnType.DECIMAL16: + case ColumnType.DECIMAL32: + case ColumnType.DECIMAL64: + return QwpConstants.TYPE_DECIMAL64; + case ColumnType.DECIMAL128: + return QwpConstants.TYPE_DECIMAL128; + case ColumnType.DECIMAL256: + return QwpConstants.TYPE_DECIMAL256; + default: + return 0; + } + } + switch (targetType) { + case ColumnType.BOOLEAN: + return QwpConstants.TYPE_BOOLEAN; + case ColumnType.BYTE: + return QwpConstants.TYPE_BYTE; + case ColumnType.SHORT: + return QwpConstants.TYPE_SHORT; + case ColumnType.CHAR: + return QwpConstants.TYPE_CHAR; + case ColumnType.INT: + return QwpConstants.TYPE_INT; + case ColumnType.LONG: + return QwpConstants.TYPE_LONG; + case ColumnType.DATE: + return QwpConstants.TYPE_DATE; + case ColumnType.FLOAT: + return QwpConstants.TYPE_FLOAT; + case ColumnType.DOUBLE: + return QwpConstants.TYPE_DOUBLE; + case ColumnType.STRING: + case ColumnType.VARCHAR: + return QwpConstants.TYPE_VARCHAR; + case ColumnType.SYMBOL: + return QwpConstants.TYPE_SYMBOL; + case ColumnType.UUID: + return QwpConstants.TYPE_UUID; + case ColumnType.LONG256: + return QwpConstants.TYPE_LONG256; + case ColumnType.IPv4: + return QwpConstants.TYPE_IPv4; + case ColumnType.BINARY: + return QwpConstants.TYPE_BINARY; + case ColumnType.TIMESTAMP_MICRO: + return QwpConstants.TYPE_TIMESTAMP; + case ColumnType.TIMESTAMP_NANO: + return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: + return 0; + } + } + + private void appendBinary( + CharSequence name, + QwpTableBuffer.ColumnBuffer column, + long ptr, + long len + ) { + if (len < 0) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary length must be non-negative"); + } + if (len > Integer.MAX_VALUE) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary length exceeds the supported maximum"); + } + if (len > 0 && ptr == 0) { + throw error(INVALID_VALUE, name, "BINARY", (int) ColumnType.BINARY, "binary pointer cannot be zero for a non-empty value"); + } + column.addBinary(ptr, len); + } + + private QwpSchemaBinding decimalColumn(CharSequence name, Decimal value, String inputType, byte inferredWireType) { + buffer.requireSchemaBinding(this); + if (value == null || value.isNull()) { + return this; + } + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + if (index < 0) { + if (!isInferredWritable(existing, name, inputType, inferredWireType)) { + return this; + } + QwpTableBuffer.ColumnBuffer inferred = claimInferred(existing, name, inferredWireType); + if (value instanceof Decimal64) { + inferred.addDecimal64((Decimal64) value); + } else if (value instanceof Decimal128) { + inferred.addDecimal128((Decimal128) value); + } else { + inferred.addDecimal256((Decimal256) value); + } + return this; + } + int targetType = targetType(index, ColumnType.DECIMAL256); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column == null) { + return this; + } + return appendDecimal(column, name, value, inputType, targetType); + } + + private QwpSchemaBinding appendDecimal( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + Decimal value, + String inputType, + int targetType + ) { + if (!ColumnType.isDecimal(targetType)) { + return decimalNonDecimalColumn(column, name, value, inputType, targetType); + } + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + try { + if (!column.addSchemaDecimal(value, targetPrecision, targetScale)) { + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw targetValueConflict(name, inputType, targetType, "decimal value cannot be rescaled exactly"); + } + return this; + } + + private QwpSchemaBinding decimalNonDecimalColumn( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + Decimal value, + String inputType, + int targetType + ) { + if (targetType == ColumnType.STRING || targetType == ColumnType.VARCHAR) { + Decimal256 decimal = decimalTextScratch; + if (decimal == null) { + decimalTextScratch = decimal = new Decimal256(); + } + value.toDecimal256(decimal); + decimal.setScale(decimal.getScale() & 0xff); + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(80); + } else { + sink.clear(); + } + decimal.toSink(sink); + column.addString(sink); + return this; + } + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + + private LineSenderSchemaException invalidRange(CharSequence name, String inputType, int targetType) { + return targetValueConflict(name, inputType, targetType, "value is outside target type range"); + } + + private void parseDecimalText(CharSequence name, CharSequence value, Decimal256 scratch, int targetType) { + try { + scratch.ofString(value); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "DECIMAL256", targetType, "invalid decimal text"); + } + } + + private QwpSchemaBinding integerColumn( + CharSequence name, + long value, + String inputType, + int inferredType, + boolean sourceNull + ) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + int targetType = targetType(index, inferredType); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType) + && !isTextTarget(targetType) + && !ColumnType.isDecimal(targetType) + && !(inferredType == ColumnType.INT && targetType == ColumnType.IPv4) + && targetType != ColumnType.DATE + && targetType != ColumnType.TIMESTAMP_MICRO + && targetType != ColumnType.TIMESTAMP_NANO) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + if (ColumnType.isDecimal(targetType)) { + appendIntegerDecimal(column, name, value, inputType, targetType, sourceNull); + return this; + } + if (sourceNull) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + if (value < Byte.MIN_VALUE || value > Byte.MAX_VALUE) { + throw invalidRange(name, inputType, targetType); + } + column.addByte((byte) value); + break; + case ColumnType.SHORT: + if (value < Short.MIN_VALUE || value > Short.MAX_VALUE) { + throw invalidRange(name, inputType, targetType); + } + column.addShort((short) value); + break; + case ColumnType.INT: + column.addInt((int) value); + break; + case ColumnType.IPv4: + if (value == Numbers.IPv4_NULL) { + column.addNull(); + } else { + column.addIPv4((int) value); + } + break; + case ColumnType.LONG: + case ColumnType.DATE: + case ColumnType.TIMESTAMP_MICRO: + case ColumnType.TIMESTAMP_NANO: + column.addLong(value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble((double) value); + break; + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatLong(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatLong(value)); + break; + default: + throw new AssertionError("unsupported integer target"); + } + return this; + } + + private void appendIntegerDecimal( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + long value, + String inputType, + int targetType, + boolean sourceNull + ) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (sourceNull) { + column.addSchemaDecimalNull(targetScale); + return; + } + try { + if (!column.addSchemaLongDecimal(value, targetPrecision, targetScale)) { + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw targetValueConflict(name, inputType, targetType, "decimal value cannot be rescaled exactly"); + } + } + + private CharSequence formatLong(long value) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(20); + } else { + sink.clear(); + } + Numbers.append(sink, value, false); + return sink; + } + + private CharSequence formatLong256(long l0, long l1, long l2, long l3) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(66); + } else { + sink.clear(); + } + sink.putAscii("0x"); + if (l3 != 0) { + Numbers.appendHex(sink, l3, false); + Numbers.appendHex(sink, l2, true); + Numbers.appendHex(sink, l1, true); + Numbers.appendHex(sink, l0, true); + } else if (l2 != 0) { + Numbers.appendHex(sink, l2, false); + Numbers.appendHex(sink, l1, true); + Numbers.appendHex(sink, l0, true); + } else if (l1 != 0) { + Numbers.appendHex(sink, l1, false); + Numbers.appendHex(sink, l0, true); + } else { + Numbers.appendHex(sink, l0, false); + } + return sink; + } + + private CharSequence formatGeoHash(long value, int precisionBits) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(60); + } else { + sink.clear(); + } + for (int bit = precisionBits - 1; bit >= 0; bit--) { + sink.putAscii(((value >>> bit) & 1) == 0 ? '0' : '1'); + } + return sink; + } + + private CharSequence formatIPv4(int value) { + StringSink sink = numericTextSink; + if (sink == null) { + numericTextSink = sink = new StringSink(15); + } else { + sink.clear(); + } + Numbers.append(sink, (value >>> 24) & 0xff); + sink.put('.'); + Numbers.append(sink, (value >>> 16) & 0xff); + sink.put('.'); + Numbers.append(sink, (value >>> 8) & 0xff); + sink.put('.'); + Numbers.append(sink, value & 0xff); + return sink; + } + + private CharSequence formatFloating(double value) { + StringSink sink = floatingTextSink; + if (sink == null) { + floatingTextSink = sink = new StringSink(24); + } else { + sink.clear(); + } + Numbers.append(sink, value); + return sink; + } + + private CharSequence formatUuid(long lo, long hi) { + StringSink sink = uuidTextSink; + if (sink == null) { + uuidTextSink = sink = new StringSink(UUID_LENGTH); + } else { + sink.clear(); + } + appendUuidHex(sink, hi >>> 32, 8); + sink.put('-'); + appendUuidHex(sink, hi >>> 16, 4); + sink.put('-'); + appendUuidHex(sink, hi, 4); + sink.put('-'); + appendUuidHex(sink, lo >>> 48, 4); + sink.put('-'); + appendUuidHex(sink, lo, 12); + return sink; + } + + private static void appendUuidHex(StringSink sink, long value, int digits) { + for (int shift = (digits - 1) * 4; shift >= 0; shift -= 4) { + int nibble = (int) ((value >>> shift) & 0xf); + sink.put((char) (nibble < 10 ? '0' + nibble : 'a' + nibble - 10)); + } + } + + private LineSenderSchemaException invalidNumericText(CharSequence name, int targetType) { + return error(INVALID_VALUE, name, "STRING", targetType, "invalid numeric text"); + } + + private void appendStringNumeric( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + switch (targetType) { + case ColumnType.BYTE: + int parsed = Numbers.parseInt(value); + if (parsed < Byte.MIN_VALUE || parsed > Byte.MAX_VALUE) { + throw NumericException.instance(); + } + column.addByte((byte) parsed); + break; + case ColumnType.SHORT: + column.addShort(parseSchemaShort(value)); + break; + case ColumnType.INT: + column.addInt(Numbers.parseInt(value)); + break; + case ColumnType.LONG: + column.addLong(parseSchemaLong(value)); + break; + case ColumnType.FLOAT: + column.addFloat(parseSchemaFloat(value)); + break; + case ColumnType.DOUBLE: + column.addDouble(parseSchemaDouble(value)); + break; + default: + throw new AssertionError("unsupported numeric target"); + } + } catch (NumericException e) { + throw invalidNumericText(name, targetType); + } + } + + private void appendStringLong256( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value + ) { + int length = value.length(); + if (length < 4 || length > 66 || (length & 1) != 0 + || value.charAt(0) != '0' || value.charAt(1) != 'x') { + throw error(INVALID_VALUE, name, "STRING", (int) ColumnType.LONG256, "invalid LONG256 text"); + } + try { + int hi = length; + int lo = Math.max(2, hi - 16); + long l0 = Numbers.parseHexLong(value, lo, hi); + hi = lo; + lo = Math.max(2, hi - 16); + long l1 = hi > 2 ? Numbers.parseHexLong(value, lo, hi) : 0; + hi = lo; + lo = Math.max(2, hi - 16); + long l2 = hi > 2 ? Numbers.parseHexLong(value, lo, hi) : 0; + hi = lo; + long l3 = hi > 2 ? Numbers.parseHexLong(value, 2, hi) : 0; + if (l0 == Long.MIN_VALUE && l1 == Long.MIN_VALUE + && l2 == Long.MIN_VALUE && l3 == Long.MIN_VALUE) { + throw NumericException.instance(); + } + column.addLong256(l0, l1, l2, l3); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", (int) ColumnType.LONG256, "invalid LONG256 text"); + } + } + + private void appendStringGeoHash( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int precision, + int targetType + ) { + int chars = Math.min(value.length(), 12); + if (chars * 5 < precision) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid GEOHASH text"); + } + try { + long hash = Numbers.parseGeoHashBase32(value, 0, chars); + column.addGeoHash(hash >>> (chars * 5 - precision), precision); + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid GEOHASH text"); + } + } + + private void appendStringDate( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + long millis = QwpSchemaTimestampParser.parseDate(value); + if (millis == Long.MIN_VALUE) { + column.addNull(); + } else { + column.addLong(millis); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid DATE text"); + } + } + + private void appendStringTimestamp( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + CharSequence value, + int targetType + ) { + try { + long micros = QwpSchemaTimestampParser.parse(value); + if (targetType == ColumnType.TIMESTAMP_NANO) { + if (micros > Long.MAX_VALUE / 1000 || micros < Long.MIN_VALUE / 1000) { + throw NumericException.instance(); + } + column.addLong(micros * 1000); + } else { + column.addLong(micros); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, "STRING", targetType, "invalid timestamp text"); + } + } + + private void appendTimestampText(QwpTableBuffer.ColumnBuffer column, long micros) { + Utf8StringSink sink = timestampTextSink; + if (sink == null) { + timestampTextSink = sink = new Utf8StringSink(32); + } else { + sink.clear(); + } + QwpSchemaTimestampFormatter.appendDateTime(sink, micros); + column.addString(sink.asAsciiCharSequence()); + } + + private static double parseSchemaDouble(CharSequence value) throws NumericException { + int length = oppositeFloatingSuffixLength(value, 'f'); + return length < 0 + ? Numbers.parseDouble(value) + : FastDoubleParser.parseDouble(value, 0, length, true); + } + + private static float parseSchemaFloat(CharSequence value) throws NumericException { + int length = oppositeFloatingSuffixLength(value, 'd'); + return length < 0 + ? FastFloatParser.parseFloat(value, true) + : FastFloatParser.parseFloat(value, 0, length, true); + } + + private static int oppositeFloatingSuffixLength(CharSequence value, char suffix) { + int index = value.length() - 1; + while (index >= 0 && value.charAt(index) <= ' ') { + index--; + } + if (index <= 0 || (value.charAt(index) | 32) != suffix) { + return -1; + } + char preceding = value.charAt(index - 1); + return (preceding >= '0' && preceding <= '9') || preceding == '.' ? index : -1; + } + + private static long parseSchemaLong(CharSequence value) throws NumericException { + int length = value.length(); + if (length == 0) { + throw NumericException.instance(); + } + boolean negative = value.charAt(0) == '-'; + int index = negative ? 1 : 0; + if (index == length) { + throw NumericException.instance(); + } + long limit = negative ? Long.MIN_VALUE : -Long.MAX_VALUE; + long multiplyLimit = limit / 10; + long result = 0; + int digitCount = 0; + for (; index < length; index++) { + char c = value.charAt(index); + if (c == '_' || c == 0x7f) { + if (digitCount == 0) { + throw NumericException.instance(); + } + digitCount = 0; + } else if (c == 'L' || c == 'l') { + if (digitCount == 0 || index + 1 != length) { + throw NumericException.instance(); + } + } else { + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + int digit = c - '0'; + if (result < multiplyLimit) { + throw NumericException.instance(); + } + result *= 10; + if (result < limit + digit) { + throw NumericException.instance(); + } + result -= digit; + digitCount++; + } + } + if (digitCount == 0) { + throw NumericException.instance(); + } + return negative ? result : -result; + } + + private static short parseSchemaShort(CharSequence value) throws NumericException { + int length = value.length(); + int index = length > 0 && value.charAt(0) == '-' ? 1 : 0; + if (index == length) { + throw NumericException.instance(); + } + for (; index < length; index++) { + char c = value.charAt(index); + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + } + int parsed = Numbers.parseInt(value); + if (parsed < Short.MIN_VALUE || parsed > Short.MAX_VALUE) { + throw NumericException.instance(); + } + return (short) parsed; + } + + private QwpSchemaBinding floatingNumericColumn(CharSequence name, double value, String inputType) { + buffer.requireSchemaBinding(this); + QwpTableBuffer.ColumnBuffer existing = existingColumn(name); + int index = targetIndex(existing, name, inputType); + int targetType = targetType(index, + "FLOAT".equals(inputType) ? ColumnType.FLOAT : ColumnType.DOUBLE); + QwpTableBuffer.ColumnBuffer column = targetColumn(existing, name, inputType, index, targetType); + if (column == null) { + return this; + } + if (!isNumericTarget(targetType)) { + return floatingNonNumericColumn(column, name, value, inputType, targetType); + } + if (Double.isNaN(value)) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.BYTE: + requireWholeInRange(name, inputType, targetType, value, Byte.MIN_VALUE, Byte.MAX_VALUE); + column.addByte((byte) value); + break; + case ColumnType.SHORT: + requireWholeInRange(name, inputType, targetType, value, Short.MIN_VALUE, Short.MAX_VALUE); + column.addShort((short) value); + break; + case ColumnType.INT: + requireWholeInRange(name, inputType, targetType, value, Integer.MIN_VALUE, Integer.MAX_VALUE); + column.addInt((int) value); + break; + case ColumnType.LONG: + if (!Double.isFinite(value) || value != Math.rint(value) || value < -0x1.0p63 || value >= 0x1.0p63) { + throw error(INVALID_VALUE, name, inputType, targetType, "value cannot be represented exactly by target integer type"); + } + column.addLong((long) value); + break; + case ColumnType.FLOAT: + column.addFloat((float) value); + break; + case ColumnType.DOUBLE: + column.addDouble(value); + break; + } + return this; + } + + private QwpSchemaBinding floatingNonNumericColumn( + QwpTableBuffer.ColumnBuffer column, + CharSequence name, + double value, + String inputType, + int targetType + ) { + if (ColumnType.isDecimal(targetType)) { + int targetPrecision = ColumnType.getDecimalPrecision(targetType); + int targetScale = ColumnType.getDecimalScale(targetType); + if (!Double.isFinite(value)) { + column.addSchemaDecimalNull(targetScale); + return this; + } + try { + if (!column.addSchemaStringDecimal(formatFloating(value), targetPrecision, targetScale)) { + throw targetValueConflict(name, inputType, targetType, "decimal value exceeds target precision"); + } + } catch (NumericException e) { + throw error(INVALID_VALUE, name, inputType, targetType, + "floating value cannot be represented exactly by target decimal precision and scale"); + } + return this; + } + if (!isTextTarget(targetType)) { + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + if (Double.isNaN(value)) { + column.addNull(); + return this; + } + switch (targetType) { + case ColumnType.STRING: + case ColumnType.VARCHAR: + column.addString(formatFloating(value)); + break; + case ColumnType.SYMBOL: + column.addSymbol(formatFloating(value)); + break; + default: + throw targetTypeConflict(name, inputType, targetType, "conversion is not implemented"); + } + return this; + } + + private void requireWholeInRange( + CharSequence name, + String inputType, + int targetType, + double value, + double min, + double max + ) { + if (!Double.isFinite(value) || value != Math.rint(value) || value < min || value > max) { + throw error(INVALID_VALUE, name, inputType, targetType, "value cannot be represented exactly by target integer type"); + } + } + + private static boolean isNumericTarget(int targetType) { + return targetType == ColumnType.BYTE + || targetType == ColumnType.SHORT + || targetType == ColumnType.INT + || targetType == ColumnType.LONG + || targetType == ColumnType.FLOAT + || targetType == ColumnType.DOUBLE; + } + + private static boolean isTextTarget(int targetType) { + return targetType == ColumnType.STRING + || targetType == ColumnType.VARCHAR + || targetType == ColumnType.SYMBOL; + } + + private static boolean isSupportedTimestampUnit(ChronoUnit unit) { + return unit == ChronoUnit.NANOS + || unit == ChronoUnit.MICROS + || unit == ChronoUnit.MILLIS + || unit == ChronoUnit.SECONDS + || unit == ChronoUnit.MINUTES + || unit == ChronoUnit.HOURS + || unit == ChronoUnit.DAYS; + } + + // Matches client UTF-8 replacement followed by the server's first-BMP decoder: + // supplementary code points become NUL and malformed surrogates become '?'. + private static char stringToChar(CharSequence value) { + if (value.length() == 0) { + return 0; + } + char first = value.charAt(0); + if (Character.isHighSurrogate(first)) { + return value.length() > 1 && Character.isLowSurrogate(value.charAt(1)) ? 0 : '?'; + } + return Character.isLowSurrogate(first) ? '?' : first; + } + + private static long instantTimestamp(Instant instant, long unitsPerSecond, long fraction) { + long seconds = instant.getEpochSecond(); + if (seconds < 0) { + // Keep both terms non-positive so an intermediate underflow also means the final sum is out of range. + return Math.addExact(Math.multiplyExact(seconds + 1, unitsPerSecond), fraction - unitsPerSecond); + } + return Math.addExact(Math.multiplyExact(seconds, unitsPerSecond), fraction); + } + + private int requireDesignatedTimestamp(String inputType) { + int index = schema.getDesignatedIndex(); + if (index < 0) { + throw unsupported(null, inputType, -1, "target table has no designated timestamp"); + } + int targetType = schema.getColumnType(index); + timestampWireType(targetType); + if (schema.hasColumnExtensionParameters(index)) { + throw unsupported(null, inputType, targetType, "parameterized target type"); + } + return index; + } + + private byte timestampWireType(int targetType) { + if (targetType == ColumnType.TIMESTAMP_MICRO) { + return QwpConstants.TYPE_TIMESTAMP; + } + if (targetType == ColumnType.TIMESTAMP_NANO) { + return QwpConstants.TYPE_TIMESTAMP_NANOS; + } + throw unsupported(null, "TIMESTAMP", targetType, "designated timestamp conversion is not implemented"); + } + + /** + * Returns the schema index of {@code name}, or -1 when the schema lacks it. + * A column this binding already resolved answers without re-validating or + * re-hashing the name: it can only match a name equal to its own valid one. + */ + private int targetIndex(QwpTableBuffer.ColumnBuffer existing, CharSequence name, String inputType) { + if (existing != null && existing.isSchemaResolved()) { + return existing.getSchemaIndex(); + } + if (name == null || !TableUtils.isValidColumnName(name, QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH)) { + throw error(INVALID_VALUE, name, inputType, null, "invalid column name"); + } + return schema.getColumnIndex(name); + } + + private int targetType(int index, int inferredType) { + return index >= 0 ? schema.getColumnType(index) : inferredType; + } + + // Rejects an input that the snapshot type of the named column cannot take. + private LineSenderSchemaException targetTypeConflict( + CharSequence name, + String inputType, + int targetType, + String detail + ) { + recordTargetTypeConflict(name); + return unsupported(name, inputType, targetType, detail); + } + + // Rejects a value that the snapshot type of the named column cannot hold, such + // as an INT overflow after the server widened the column to LONG. + private LineSenderSchemaException targetValueConflict( + CharSequence name, + String inputType, + int targetType, + String detail + ) { + recordTargetTypeConflict(name); + return error(INVALID_VALUE, name, inputType, targetType, detail); + } + + private LineSenderSchemaException unsupported(CharSequence name, String inputType, int targetType, String detail) { + return error(UNSUPPORTED_FEATURE, name, inputType, targetType, detail); + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java new file mode 100644 index 000000000..a9e35dd57 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaProtocol.java @@ -0,0 +1,276 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.cairo.TableUtils; +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; +import io.questdb.client.std.Unsafe; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; + +/** Codec and wire constants for negotiated QWP schema control messages. */ +public final class QwpSchemaProtocol { + public static final byte FLAG_CONTROL = 0x20; + public static final byte KIND_DESCRIBE = 1; + public static final byte KIND_SCHEMA = 2; + public static final int MAX_COLUMN_COUNT = 2048; + public static final int MAX_FRAME_SIZE = 1024 * 1024; + public static final int MAX_NAME_UTF16_LENGTH = 127; + public static final int MAX_NAME_UTF8_LENGTH = 381; + public static final int MAX_PARAMS_LENGTH = 1024; + public static final int RESULT_DENIED = 2; + public static final int RESULT_KNOWN = 0; + public static final int RESULT_MISSING = 1; + public static final int RESULT_TOO_LARGE = 4; + public static final int RESULT_UNAVAILABLE = 3; + + private static final int KNOWN_FIXED_PAYLOAD_SIZE = 1 + 8 + 1 + 4 + 8 + 2 + 2; + private static final int MIN_PAYLOAD_SIZE = 1 + 8 + 1; + + private QwpSchemaProtocol() { + } + + public static QwpSchemaResponse decodeResponse(long address, int length) { + int payloadLength = validateResponseEnvelope(address, length, MIN_PAYLOAD_SIZE); + return decodePayload(address + QwpConstants.HEADER_SIZE, payloadLength, false); + } + + /** Validates a complete schema-response envelope and returns its positive request ID. */ + public static long peekResponseRequestId(long address, int length) { + validateResponseEnvelope(address, length, 1 + Long.BYTES); + long payloadAddress = address + QwpConstants.HEADER_SIZE; + if (getUnsignedByte(payloadAddress) != KIND_SCHEMA) { + throw malformed("unexpected control response kind"); + } + long requestId = getLong(payloadAddress + 1); + if (requestId <= 0) { + throw malformed("request id must be positive"); + } + return requestId; + } + + private static int validateResponseEnvelope(long address, int length, int minimumPayloadSize) { + if (address == 0) { + throw malformed("null frame address"); + } + if (length < QwpConstants.HEADER_SIZE + minimumPayloadSize || length > MAX_FRAME_SIZE) { + throw malformed("invalid frame length: " + length); + } + if (getInt(address) != QwpConstants.MAGIC_MESSAGE) { + throw malformed("invalid QWP magic"); + } + if (getUnsignedByte(address + 4) != 1) { + throw malformed("unsupported QWP version"); + } + if (getUnsignedByte(address + 5) != (FLAG_CONTROL & 0xff)) { + throw malformed("invalid control flags"); + } + if (getUnsignedShort(address + 6) != 0) { + throw malformed("control frame must have zero tables"); + } + int payloadLength = getInt(address + 8); + if (payloadLength != length - QwpConstants.HEADER_SIZE) { + throw malformed("payload length mismatch"); + } + return payloadLength; + } + + /** Decodes a length-delimited schema payload carried by response feedback. */ + public static QwpSchemaResponse decodeFeedbackPayload(long address, int length) { + if (address == 0) { + throw malformed("null payload address"); + } + if (length < MIN_PAYLOAD_SIZE || length > MAX_FRAME_SIZE) { + throw malformed("invalid payload length: " + length); + } + return decodePayload(address, length, true); + } + + private static QwpSchemaResponse decodePayload(long address, int length, boolean feedback) { + long p = address; + long limit = address + length; + if (getUnsignedByte(p++) != KIND_SCHEMA) { + throw malformed("unexpected control response kind"); + } + long requestId = getLong(p); + p += 8; + if (feedback ? requestId != 0 : requestId <= 0) { + throw malformed(feedback ? "feedback request id must be zero" : "request id must be positive"); + } + int result = getUnsignedByte(p++); + if (result < RESULT_KNOWN || result > RESULT_TOO_LARGE) { + throw malformed("unknown schema result: " + result); + } + if (result != RESULT_KNOWN) { + if (p != limit) { + throw malformed("trailing bytes after schema result"); + } + return new QwpSchemaResponse(requestId, result, -1, -1, -1, new QwpSchemaResponse.Column[0], + new LowerCaseCharSequenceIntHashMap()); + } + if (length < KNOWN_FIXED_PAYLOAD_SIZE) { + throw malformed("truncated known schema"); + } + require(p, 16, limit, "schema identity"); + int tableId = getInt(p); + p += 4; + long metadataVersion = getLong(p); + p += 8; + int designatedIndex = getShort(p); + p += 2; + int columnCount = getUnsignedShort(p); + p += 2; + if (tableId < 0 || metadataVersion < 0) { + throw malformed("negative schema identity"); + } + if (columnCount > MAX_COLUMN_COUNT) { + throw malformed("too many columns: " + columnCount); + } + if (designatedIndex < -1 || designatedIndex >= columnCount) { + throw malformed("invalid designated index: " + designatedIndex); + } + + QwpSchemaResponse.Column[] columns = new QwpSchemaResponse.Column[columnCount]; + LowerCaseCharSequenceIntHashMap columnNames = new LowerCaseCharSequenceIntHashMap(columnCount); + for (int i = 0; i < columnCount; i++) { + require(p, 2, limit, "column name length"); + int nameLength = getUnsignedShort(p); + p += 2; + if (nameLength == 0 || nameLength > MAX_NAME_UTF8_LENGTH) { + throw malformed("invalid column name length: " + nameLength); + } + require(p, nameLength, limit, "column name"); + String name = decodeUtf8(p, nameLength, "column name"); + if (!TableUtils.isValidColumnName(name, MAX_NAME_UTF16_LENGTH)) { + throw malformed("invalid column name"); + } + if (!columnNames.put(name, i)) { + throw malformed("duplicate case-insensitive column name"); + } + p += nameLength; + require(p, 6, limit, "column type and parameter length"); + int type = getInt(p); + p += 4; + int paramsLength = getUnsignedShort(p); + p += 2; + if (paramsLength > MAX_PARAMS_LENGTH) { + throw malformed("column parameters too large: " + paramsLength); + } + require(p, paramsLength, limit, "column parameters"); + p += paramsLength; + columns[i] = new QwpSchemaResponse.Column(name, type, paramsLength != 0); + } + if (p != limit) { + throw malformed("trailing bytes after schema"); + } + return new QwpSchemaResponse(requestId, result, tableId, metadataVersion, designatedIndex, columns, columnNames); + } + + public static byte[] encodeDescribe(long requestId, CharSequence tableName) { + if (requestId <= 0) { + throw new IllegalArgumentException("request id must be positive"); + } + if (tableName == null || !TableUtils.isValidTableName(tableName, MAX_NAME_UTF16_LENGTH)) { + throw new IllegalArgumentException("invalid table name"); + } + byte[] name = encodeName(tableName, "table name"); + int payloadLength = 1 + 8 + 2 + name.length; + ByteBuffer buffer = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(QwpConstants.MAGIC_MESSAGE); + buffer.put((byte) 1); + buffer.put(FLAG_CONTROL); + buffer.putShort((short) 0); + buffer.putInt(payloadLength); + buffer.put(KIND_DESCRIBE); + buffer.putLong(requestId); + buffer.putShort((short) name.length); + buffer.put(name); + return buffer.array(); + } + + private static byte[] copyBytes(long address, int length) { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(address + i); + } + return bytes; + } + + private static String decodeUtf8(long address, int length, String what) { + byte[] bytes = copyBytes(address, length); + try { + return StandardCharsets.UTF_8.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw malformed("invalid UTF-8 " + what); + } + } + + private static byte[] encodeName(CharSequence value, String what) { + if (value == null || value.length() == 0 || value.length() > MAX_NAME_UTF16_LENGTH) { + throw new IllegalArgumentException(what + " must contain 1 to " + MAX_NAME_UTF16_LENGTH + " UTF-16 units"); + } + byte[] bytes = value.toString().getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_NAME_UTF8_LENGTH) { + throw new IllegalArgumentException(what + " exceeds " + MAX_NAME_UTF8_LENGTH + " UTF-8 bytes"); + } + return bytes; + } + + private static int getInt(long address) { + return Unsafe.getUnsafe().getInt(address); + } + + private static long getLong(long address) { + return Unsafe.getUnsafe().getLong(address); + } + + private static short getShort(long address) { + return Unsafe.getUnsafe().getShort(address); + } + + private static int getUnsignedByte(long address) { + return Unsafe.getUnsafe().getByte(address) & 0xff; + } + + private static int getUnsignedShort(long address) { + return getShort(address) & 0xffff; + } + + private static IllegalArgumentException malformed(String message) { + return new IllegalArgumentException("Malformed QWP schema response: " + message); + } + + private static void require(long position, int needed, long limit, String what) { + if (needed < 0 || position > limit || needed > limit - position) { + throw malformed("truncated " + what); + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java new file mode 100644 index 000000000..49fa5883d --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaResponse.java @@ -0,0 +1,122 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.LowerCaseCharSequenceIntHashMap; + +/** + * Immutable response to a QWP schema DESCRIBE control request. The snapshot + * owns the case-insensitive column-name index that the decoder builds while + * it validates name uniqueness, so consumers never rebuild it. + */ +public final class QwpSchemaResponse { + private final LowerCaseCharSequenceIntHashMap columnIndexes; + private final Column[] columns; + private final int designatedIndex; + private final int result; + private final int tableId; + private final long metadataVersion; + private final long requestId; + + QwpSchemaResponse( + long requestId, + int result, + int tableId, + long metadataVersion, + int designatedIndex, + Column[] columns, + LowerCaseCharSequenceIntHashMap columnIndexes + ) { + this.requestId = requestId; + this.result = result; + this.tableId = tableId; + this.metadataVersion = metadataVersion; + this.designatedIndex = designatedIndex; + this.columns = columns; + this.columnIndexes = columnIndexes; + } + + public int getColumnCount() { + return columns.length; + } + + /** Returns the index of the case-insensitively named column, or -1. */ + public int getColumnIndex(CharSequence name) { + return columnIndexes.get(name); + } + + public String getColumnName(int index) { + return columns[index].name; + } + + public int getColumnType(int index) { + return columns[index].type; + } + + /** + * Returns whether the server supplied an extension parameter block for + * this column's type. The current client safely skips those bytes but must + * not interpret a parameterized type as an unparameterized one. + */ + public boolean hasColumnExtensionParameters(int index) { + return columns[index].hasTypeParameters; + } + + public int getDesignatedIndex() { + return designatedIndex; + } + + public long getMetadataVersion() { + return metadataVersion; + } + + /** Returns the positive DESCRIBE id, or zero for named write feedback. */ + public long getRequestId() { + return requestId; + } + + public int getResult() { + return result; + } + + public int getTableId() { + return tableId; + } + + public boolean hasSchema() { + return result == QwpSchemaProtocol.RESULT_KNOWN; + } + + static final class Column { + private final boolean hasTypeParameters; + private final String name; + private final int type; + + Column(String name, int type, boolean hasTypeParameters) { + this.name = name; + this.type = type; + this.hasTypeParameters = hasTypeParameters; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java new file mode 100644 index 000000000..05ff6c7ae --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java @@ -0,0 +1,187 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.Numbers; +import io.questdb.client.std.datetime.CommonUtils; +import io.questdb.client.std.str.Utf8StringSink; + +/** Schema-only UTC millisecond formatter matching the server timestamp-to-text conversion. */ +final class QwpSchemaTimestampFormatter { + private static final long DAY_MICROS = 86_400_000_000L; + private static final long AVG_YEAR_MICROS = (long) (365.2425 * DAY_MICROS); + private static final long HOUR_MICROS = 3_600_000_000L; + private static final long MILLI_MICROS = 1_000L; + private static final long MINUTE_MICROS = 60_000_000L; + private static final long SECOND_MICROS = 1_000_000L; + private static final long YEAR_MICROS_LEAP = 366 * DAY_MICROS; + private static final long YEAR_MICROS_NONLEAP = 365 * DAY_MICROS; + private static final long[] LEAP_MONTH_MICROS = new long[12]; + private static final long[] MONTH_MICROS = new long[12]; + + static { + long normal = 0; + long leap = 0; + for (int month = 0; month < 11; month++) { + normal += CommonUtils.DAYS_PER_MONTH[month] * DAY_MICROS; + leap += CommonUtils.getDaysPerMonth(month + 1, true) * DAY_MICROS; + MONTH_MICROS[month + 1] = normal; + LEAP_MONTH_MICROS[month + 1] = leap; + } + } + + private QwpSchemaTimestampFormatter() { + } + + static void appendDateTime(Utf8StringSink sink, long micros) { + if (micros == Long.MIN_VALUE) { + return; + } + int year = getYear(micros); + boolean leap = CommonUtils.isLeapYear(year); + int month = getMonthOfYear(micros, year, leap); + appendYear000(sink, year); + sink.putAscii('-'); + append0(sink, month); + sink.putAscii('-'); + append0(sink, getDayOfMonth(micros, year, month, leap)); + sink.putAscii('T'); + append0(sink, getHourOfDay(micros)); + sink.putAscii(':'); + append0(sink, getMinuteOfHour(micros)); + sink.putAscii(':'); + append0(sink, getSecondOfMinute(micros)); + sink.putAscii('.'); + append00(sink, getMillisOfSecond(micros)); + sink.putAscii('Z'); + } + + private static void append0(Utf8StringSink sink, int value) { + if (Math.abs(value) < 10) { + sink.putAscii('0'); + } + Numbers.append(sink, value); + } + + private static void append00(Utf8StringSink sink, int value) { + int absolute = Math.abs(value); + if (absolute < 10) { + sink.putAscii('0').putAscii('0'); + } else if (absolute < 100) { + sink.putAscii('0'); + } + Numbers.append(sink, value); + } + + private static void appendYear000(Utf8StringSink sink, int value) { + int absolute = Math.abs(value); + if (absolute < 10) { + sink.putAscii('0').putAscii('0').putAscii('0'); + } else if (absolute < 100) { + sink.putAscii('0').putAscii('0'); + } else if (absolute < 1000) { + sink.putAscii('0'); + } + Numbers.append(sink, value != 0 ? value : 1); + } + + private static int getDayOfMonth(long micros, int year, int month, boolean leap) { + long start = yearMicros(year, leap) + monthOfYearMicros(month, leap); + return (int) ((micros - start) / DAY_MICROS) + 1; + } + + private static int getHourOfDay(long micros) { + if (micros > -1) { + return (int) ((micros / HOUR_MICROS) % 24); + } + return 23 + (int) (((micros + 1) / HOUR_MICROS) % 24); + } + + private static int getMillisOfSecond(long micros) { + if (micros > -1) { + return (int) ((micros / MILLI_MICROS) % 1000); + } + return 999 + (int) (((micros + 1) / MILLI_MICROS) % 1000); + } + + private static int getMinuteOfHour(long micros) { + if (micros > -1) { + return (int) ((micros / MINUTE_MICROS) % 60); + } + return 59 + (int) (((micros + 1) / MINUTE_MICROS) % 60); + } + + // Keep the server's threshold-based month lookup: it is stable at the wrapped long extrema. + private static int getMonthOfYear(long micros, int year, boolean leap) { + int i = (int) (((micros - yearMicros(year, leap)) / 1000) >> 10); + if (leap) { + if (i < 182 * 84375) { + if (i < 91 * 84375) { + return i < 31 * 84375 ? 1 : i < 60 * 84375 ? 2 : 3; + } + return i < 121 * 84375 ? 4 : i < 152 * 84375 ? 5 : 6; + } + if (i < 274 * 84375) { + return i < 213 * 84375 ? 7 : i < 244 * 84375 ? 8 : 9; + } + return i < 305 * 84375 ? 10 : i < 335 * 84375 ? 11 : 12; + } + if (i < 181 * 84375) { + if (i < 90 * 84375) { + return i < 31 * 84375 ? 1 : i < 59 * 84375 ? 2 : 3; + } + return i < 120 * 84375 ? 4 : i < 151 * 84375 ? 5 : 6; + } + if (i < 273 * 84375) { + return i < 212 * 84375 ? 7 : i < 243 * 84375 ? 8 : 9; + } + return i < 304 * 84375 ? 10 : i < 334 * 84375 ? 11 : 12; + } + + private static int getSecondOfMinute(long micros) { + if (micros > -1) { + return (int) ((micros / SECOND_MICROS) % 60); + } + return 59 + (int) (((micros + 1) / SECOND_MICROS) % 60); + } + + // Source-compatible estimate and adjustment from the server Micros implementation. + private static int getYear(long micros) { + int yearsSinceEpoch = (int) (micros / AVG_YEAR_MICROS); + int estimate = 1970 + yearsSinceEpoch; + if (micros < 0 && estimate >= 1970) { + estimate = 1969; + } + boolean leap = CommonUtils.isLeapYear(estimate); + long start = yearMicros(estimate, leap); + long difference = micros - start; + if (difference < 0) { + estimate--; + } else if (difference >= (leap ? YEAR_MICROS_LEAP : YEAR_MICROS_NONLEAP)) { + estimate++; + } + return estimate; + } + + private static long monthOfYearMicros(int month, boolean leap) { + return (leap ? LEAP_MONTH_MICROS : MONTH_MICROS)[month - 1]; + } + + private static long yearMicros(int year, boolean leap) { + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + long days = year * 365L + (leapYears - 719527L); + long micros = days * DAY_MICROS; + return days < 0 && micros > 0 ? Long.MIN_VALUE : micros; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java new file mode 100644 index 000000000..ed1a57b3b --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java @@ -0,0 +1,752 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.cutlass.qwp.protocol; + +import io.questdb.client.std.NumericException; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.datetime.CommonUtils; + +import java.text.DateFormatSymbols; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.zone.ZoneOffsetTransition; +import java.time.zone.ZoneOffsetTransitionRule; +import java.time.zone.ZoneRules; +import java.time.zone.ZoneRulesProvider; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Source-faithful parser for the fixed formats accepted by QWP STRING-to-timestamp + * and STRING-to-DATE conversion. The grammar and computation mirror the server's + * {@code MicrosFormatUtils}, {@code DateFormatUtils} and generic format parsers; + * this is deliberately not a general date-format compiler. + */ +final class QwpSchemaTimestampParser { + private static final int DAYS_0000_TO_1970 = 719527; + private static final long GREEDY_MILLIS_FAILED = -1L; + private static final int[] MONTH_START_DAYS_COMMON = monthStartDays(false); + private static final int[] MONTH_START_DAYS_LEAP = monthStartDays(true); + private static final int REFERENCE_CENTURY = referenceCentury(); + // The server parses TIMESTAMP text with MicrosFormatUtils over UTF-8 bytes, + // and DATE text with DateFormatUtils over UTF-16. Its two parsers also differ + // in year overflow and in zone rules for negative years. + private static final Target DATE = new Target(1L, false, false, true); + private static final Target TIMESTAMP = new Target(1_000L, true, true, false); + + private QwpSchemaTimestampParser() { + } + + /** + * Parses TIMESTAMP text to epoch micros. The server parses the UTF-8 bytes, + * where no non-ASCII byte matches anything, so any non-ASCII character fails. + */ + static long parse(CharSequence value) throws NumericException { + // Only the PG layout has a space after the day, and a space there fails + // every ISO layout, so that one character selects the parser. + int n = value.length(); + int dash = firstDash(value, n); + if (dash >= 1 && dash + 6 < n && value.charAt(dash + 6) == ' ') { + return parsePg(value, dash); + } + return parseIso(value); + } + + /** Parses DATE text to epoch millis; the server parses DATE text as UTF-16. */ + static long parseDate(CharSequence value) throws NumericException { + int n = value.length(); + int dash = firstDash(value, n); + if (dash < 1) { + // Both date layouts need an inner dash and a raw number cannot hold one. + return Numbers.parseLong(value); + } + // PG ends at the day or continues with a space; only UTC has a 'T' there. + int separator = dash + 6; + if (separator >= n || value.charAt(separator) == ' ') { + return parsePgDate(value, dash); + } + if (value.charAt(separator) == 'T') { + return parseUtcDate(value); + } + throw NumericException.instance(); + } + + private static int firstDash(CharSequence value, int n) { + return indexOf(value, '-', n > 0 && value.charAt(0) == '-' ? 1 : 0, n); + } + + private static long parsePgDate(CharSequence value, int dash) throws NumericException { + int n = value.length(); + int year = greedyInt(value, 0, dash); + if (dash == 2) { + year += REFERENCE_CENTURY; + } + int pos = dash + 1; + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(DATE, year, month, day, 0, 0, 0, 0, 0); + } + require(value, pos++, n, ' '); + int tail = pos; + try { + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + int millis = 0; + if (pos < n && value.charAt(pos) == '.') { + long parsed = greedyMillis(value, pos + 1, n); + if (parsed == GREEDY_MILLIS_FAILED) { + throw NumericException.instance(); + } + millis = (int) parsed; + pos += 1 + (int) (parsed >>> 32); + } + return zoned(DATE, localTime(DATE, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + } catch (NumericException ignored) { + return zoned(DATE, localTime(DATE, year, month, day, 0, 0, 0, 0, 0), year, value, tail, n); + } + } + + private static long parseUtcDate(CharSequence value) throws NumericException { + int n = value.length(); + int pos = 0; + int year = fixedInt(value, pos, pos += 4, n); + require(value, pos++, n, '-'); + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, 'T'); + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '.'); + int millis = fixedInt(value, pos, pos += 3, n); + return zoned(DATE, localTime(DATE, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + } + + private static long parseIso(CharSequence value) throws NumericException { + int n = value.length(); + int pos = n > 0 && value.charAt(0) == '-' ? 5 : 4; + int year = pos == 5 ? -fixedInt(value, 1, pos, n) : fixedInt(value, 0, pos, n); + if (pos == n) { + return localTime(TIMESTAMP, year, 1, 1, 0, 0, 0, 0, 0); + } + require(value, pos++, n, '-'); + if (pos + 3 <= n && value.charAt(pos) == 'W') { + int week = Numbers.parseInt(value, pos + 1, pos + 3); + if (pos + 3 != n) { + throw NumericException.instance(); + } + return weekMicros(year, week); + } + int month = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(TIMESTAMP, year, month, 1, 0, 0, 0, 0, 0); + } + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(TIMESTAMP, year, month, day, 0, 0, 0, 0, 0); + } + require(value, pos++, n, 'T'); + int hour = fixedInt(value, pos, pos += 2, n); + if (pos == n) { + return localTime(TIMESTAMP, year, month, day, hour, 0, 0, 0, 0); + } + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + if (pos < n && value.charAt(pos) == '.') { + return parseIsoFraction(value, pos + 1, n, year, month, day, hour, minute, second); + } + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, 0, 0), year, value, pos, n); + } + + /** + * Tries the server's fraction layouts in its order: greedy S to SSS, then + * SSSUUU, then S followed by greedy S to SSS, then fixed SSS. A layout whose + * fraction, fields or zone fails hands over to the next one. + */ + private static long parseIsoFraction( + CharSequence value, + int pos, + int n, + int year, + int month, + int day, + int hour, + int minute, + int second + ) throws NumericException { + long parsed = greedyMillis(value, pos, n); + if (parsed != GREEDY_MILLIS_FAILED) { + try { + long local = localTime(TIMESTAMP, year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(TIMESTAMP, local, year, value, pos + (int) (parsed >>> 32), n); + } catch (NumericException ignored) { + } + } + if (pos + 6 <= n) { + try { + int millis = fixedInt(value, pos, pos + 3, n); + int micros = fixedInt(value, pos + 3, pos + 6, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, micros), year, value, pos + 6, n); + } catch (NumericException ignored) { + } + } + if (pos + 2 <= n) { + try { + fixedInt(value, pos, pos + 1, n); + parsed = greedyMillis(value, pos + 1, n); + if (parsed != GREEDY_MILLIS_FAILED) { + long local = localTime(TIMESTAMP, year, month, day, hour, minute, second, (int) parsed, 0); + return zoned(TIMESTAMP, local, year, value, pos + 1 + (int) (parsed >>> 32), n); + } + } catch (NumericException ignored) { + } + } + // Final UTC_PATTERN fallback is fixed SSS. It intentionally follows + // the greedy S and SS forms, and therefore accepts parseInt forms + // such as +12 and 1_2 as well as a numeric-zone suffix after 3 chars. + if (pos + 3 <= n) { + try { + int millis = fixedInt(value, pos, pos + 3, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, 0), year, value, pos + 3, n); + } catch (NumericException ignored) { + } + } + throw NumericException.instance(); + } + + private static long parsePg(CharSequence value, int dash) throws NumericException { + int n = value.length(); + int year = greedyInt(value, 0, dash); + if (dash == 2) { + year += REFERENCE_CENTURY; + } + int pos = dash + 1; + int month = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '-'); + int day = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ' '); + int hour = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int minute = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, ':'); + int second = fixedInt(value, pos, pos += 2, n); + require(value, pos++, n, '.'); + int millis = fixedInt(value, pos, pos += 3, n); + return zoned(TIMESTAMP, localTime(TIMESTAMP, year, month, day, hour, minute, second, millis, 0), year, value, pos, n); + } + + /** + * Returns {@code (width << 32) | millis} for a greedy 1-3 digit fraction with + * an optional minus sign, or {@link #GREEDY_MILLIS_FAILED}. + */ + private static long greedyMillis(CharSequence value, int lo, int hi) { + int p = lo; + boolean negative = p < hi && value.charAt(p) == '-'; + if (negative) { + p++; + } + int digitStart = p; + int parsed = 0; + while (p < hi && value.charAt(p) >= '0' && value.charAt(p) <= '9') { + parsed = parsed * 10 + value.charAt(p++) - '0'; + } + int width = p - lo; + if (p == digitStart || width > 3) { + return GREEDY_MILLIS_FAILED; + } + while (width < 3) { + parsed *= 10; + width++; + } + return ((long) (p - lo) << 32) | ((negative ? -parsed : parsed) & 0xffffffffL); + } + + /** Local time in {@code t}'s unit, from validated fields. */ + private static long localTime( + Target t, + int year, + int month, + int day, + int hour, + int minute, + int second, + int millis, + int micros + ) throws NumericException { + boolean leap = CommonUtils.isLeapYear(year); + if (month < 1 || month > 12 || day < 1 || day > CommonUtils.getDaysPerMonth(month, leap) + || hour < 0 || hour > 24 || minute < 0 || minute > 59 + || second < 0 || second > 59) { + throw NumericException.instance(); + } + return yearStart(t, year, leap) + monthStart(t, month, leap) + (day - 1L) * t.day + + (hour % 24L) * t.hour + minute * t.minute + second * t.second + + millis * t.milli + micros; + } + + private static long weekMicros(int year, int week) throws NumericException { + if (week == -1) { + // The server's "no week" sentinel is -1, so an explicit W-1 parses + // as the bare year. + return localTime(TIMESTAMP, year, 1, 1, 0, 0, 0, 0, 0); + } + if (week < 1 || week > weeks(year)) { + throw NumericException.instance(); + } + boolean leap = CommonUtils.isLeapYear(year); + long first = yearStart(TIMESTAMP, year, leap) + (week - 1L) * 7 * TIMESTAMP.day + + isoYearDayOffset(year) * TIMESTAMP.day; + int actualYear = year(first); + int month = monthOfYear(first, actualYear, CommonUtils.isLeapYear(actualYear)); + int weekYear = year + (week == 1 && isoYearDayOffset(year) < 0 ? -1 : 0); + int day = dayOfMonth(first, weekYear, month, CommonUtils.isLeapYear(weekYear)); + // GenericMicrosFormat retains the originally parsed year's leap flag + // after an ISO week crosses a calendar-year boundary. + return yearStart(TIMESTAMP, weekYear, leap) + monthStart(TIMESTAMP, month, leap) + (day - 1L) * TIMESTAMP.day; + } + + /** Converts {@code local}, in {@code t}'s unit, to UTC using the zone text in {@code [lo, hi)}. */ + private static long zoned(Target t, long local, int year, CharSequence value, int lo, int hi) throws NumericException { + return local - ZoneMatcher.offset(t, value, lo, hi, local, year); + } + + private static int fixedInt(CharSequence s, int lo, int hi, int limit) throws NumericException { + if (hi > limit) { + throw NumericException.instance(); + } + return Numbers.parseInt(s, lo, hi); + } + + private static int greedyInt(CharSequence s, int lo, int hi) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); + } + int sign = 1; + if (s.charAt(lo) == '-') { + sign = -1; + lo++; + } + if (lo >= hi) { + throw NumericException.instance(); + } + long value = 0; + for (int i = lo; i < hi; i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + throw NumericException.instance(); + } + value = value * 10 + c - '0'; + if (value > (sign < 0 ? 2147483648L : Integer.MAX_VALUE)) { + throw NumericException.instance(); + } + } + return (int) (sign * value); + } + + private static int indexOf(CharSequence s, char c, int lo, int hi) { + for (int i = lo; i < hi; i++) { + if (s.charAt(i) == c) { + return i; + } + } + return -1; + } + + private static void require(CharSequence s, int pos, int hi, char c) throws NumericException { + if (pos >= hi || s.charAt(pos) != c) { + throw NumericException.instance(); + } + } + + private static int referenceCentury() { + int year = java.time.ZonedDateTime.now(ZoneOffset.UTC).getYear(); + int offset = year % 100; + return offset + 20 > 100 ? year - offset + 100 : year - offset; + } + + private static int[] monthStartDays(boolean leap) { + int[] result = new int[12]; + for (int i = 1; i < 12; i++) { + result[i] = result[i - 1] + CommonUtils.getDaysPerMonth(i, leap); + } + return result; + } + private static long monthStart(Target t, int month, boolean leap) { + return (leap ? MONTH_START_DAYS_LEAP : MONTH_START_DAYS_COMMON)[month - 1] * t.day; + } + private static long yearStart(Target t, int year, boolean leap) { + // Preserve the server's unchecked multiplication. Only its micros + // variant saturates on negative overflow, and asymmetrically. + int leapYears = year / 100; + if (year < 0) { + leapYears = ((year + 3) >> 2) - leapYears + ((leapYears + 3) >> 2) - 1; + } else { + leapYears = (year >> 2) - leapYears + (leapYears >> 2); + if (leap) { + leapYears--; + } + } + long days = year * 365L + leapYears - DAYS_0000_TO_1970; + long start = days * t.day; + return t.isYearSaturating && days < 0 && start > 0 ? Long.MIN_VALUE : start; + } + private static int dayOfWeek(Target t, long time) { + long d; + if (time > -1) { + d = time / t.day; + } else { + d = (time - (t.day - 1)) / t.day; + if (d < -3) { + return 7 + (int) ((d + 4) % 7); + } + } + return 1 + (int) ((d + 3) % 7); + } + private static int endYearDow(int year) { + return (year + Math.abs(year / 4) - Math.abs(year / 100) + Math.abs(year / 400)) % 7; + } + + private static int weeks(int year) { + return endYearDow(year) == 4 || endYearDow(year - 1) == 3 ? 53 : 52; + } + + private static int isoYearDayOffset(int year) { + int d = endYearDow(year - 1); + return (d <= 3 ? 0 : 7) - d; + } + + private static int monthOfYear(long micros, int year, boolean leap) { + long elapsed = micros - yearStart(TIMESTAMP, year, leap); + int month = 12; + while (month > 1 && elapsed < monthStart(TIMESTAMP, month, leap)) { + month--; + } + return month; + } + private static int year(long micros) { + int estimate = 1970 + (int) (micros / 31_556_952_000_000L); + if (micros < 0 && estimate >= 1970) { + estimate = 1969; + } + boolean leap = CommonUtils.isLeapYear(estimate); + long diff = micros - yearStart(TIMESTAMP, estimate, leap); + if (diff < 0) { + return estimate - 1; + } + return diff >= (leap ? 366L : 365L) * TIMESTAMP.day ? estimate + 1 : estimate; + } + private static int dayOfMonth(long micros, int year, int month, boolean leap) { + return (int) ((micros - yearStart(TIMESTAMP, year, leap) - monthStart(TIMESTAMP, month, leap)) / TIMESTAMP.day) + 1; + } + private static long nextOrSame(Target t, long time, int dow) { + int current = dayOfWeek(t, time); + return current <= dow ? time + (dow - current) * t.day : time + (7 - current + dow) * t.day; + } + private static long previousOrSame(Target t, long time, int dow) { + int current = dayOfWeek(t, time); + return current >= dow ? time - (current - dow) * t.day : time - (7 + current - dow) * t.day; + } + private static final class ZoneMatcher { + + /** + * Zone offset in {@code t}'s unit. The server matches TIMESTAMP zone text + * against UTF-8 bytes, where a non-ASCII name never matches, and DATE zone + * text case-insensitively as UTF-16. + */ + private static long offset(Target t, CharSequence s, int lo, int hi, long epoch, int year) throws NumericException { + if (lo >= hi) { + throw NumericException.instance(); + } + int minutes = numericOffset(s, lo, hi); + if (minutes != Integer.MIN_VALUE) { + return minutes * t.minute; + } + return NamedZones.find(s, lo, hi, t.isZoneAsciiOnly).offset(t, epoch, year); + } + + private static int numericOffset(CharSequence s, int lo, int hi) { + if (hi - lo == 1 && (s.charAt(lo) == 'Z' || s.charAt(lo) == 'z')) { + return 0; + } + int p = lo; + boolean namedPrefix = startsWithIgnoreCase(s, p, hi, "UTC") || startsWithIgnoreCase(s, p, hi, "GMT"); + if (namedPrefix) { + p += 3; + } + boolean negative = false; + if (p < hi && (s.charAt(p) == '+' || s.charAt(p) == '-')) { + negative = s.charAt(p++) == '-'; + } else if (namedPrefix) { + return Integer.MIN_VALUE; + } + if (p + 2 > hi || !digit(s.charAt(p)) || !digit(s.charAt(p + 1))) { + return Integer.MIN_VALUE; + } + int hour = (s.charAt(p) - '0') * 10 + s.charAt(p + 1) - '0'; + p += 2; + int minute = 0; + if (p < hi) { + if (s.charAt(p) == ':') { + p++; + } + if (p + 2 > hi || !digit(s.charAt(p)) || !digit(s.charAt(p + 1))) { + return Integer.MIN_VALUE; + } + minute = (s.charAt(p) - '0') * 10 + s.charAt(p + 1) - '0'; + p += 2; + } + return p == hi && hour <= 23 && minute <= 59 + ? (negative ? -1 : 1) * (hour * 60 + minute) + : Integer.MIN_VALUE; + } + + private static boolean digit(char c) { + return c >= '0' && c <= '9'; + } + + private static boolean startsWithIgnoreCase(CharSequence value, int lo, int hi, String prefix) { + if (hi - lo < prefix.length()) { + return false; + } + for (int i = 0; i < prefix.length(); i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != prefix.charAt(i)) { + return false; + } + } + return true; + } + + /** + * Zone names by case-insensitive text. A holder class, so that only a + * named-zone lookup loads every zone's rules; 'Z' and numeric offsets + * never initialize it. + */ + private static final class NamedZones { + private static final int MASK; + private static final String[] NAMES; + private static final Zone[] ZONES; + + static { + Map rules = new HashMap<>(); + for (String id : ZoneRulesProvider.getAvailableZoneIds()) { + rules.put(id, new Zone(ZoneRulesProvider.getRules(id, true))); + } + for (Map.Entry alias : ZoneId.SHORT_IDS.entrySet()) { + if (!rules.containsKey(alias.getKey())) { + Zone target = rules.get(alias.getValue()); + if (target == null) { + target = new Zone(ZoneId.of(alias.getValue()).getRules()); + } + rules.put(alias.getKey(), target); + } + } + String[][] rows = new DateFormatSymbols(Locale.ENGLISH).getZoneStrings(); + int names = 1; + for (String[] row : rows) { + names += row.length; + } + // At most half full, so a miss ends at an empty slot quickly. + int capacity = 16; + while (capacity < 2 * names) { + capacity <<= 1; + } + MASK = capacity - 1; + NAMES = new String[capacity]; + ZONES = new Zone[capacity]; + // The first definition of a name wins; UTC goes first. + define(rules.get("UTC"), "UTC"); + for (String[] row : rows) { + if (row.length == 0 || !rules.containsKey(row[0])) { + continue; + } + Zone zone = rules.get(row[0]); + for (String name : row) { + define(zone, name); + } + } + } + + private static void define(Zone zone, String name) { + if (zone == null || name == null || name.isEmpty()) { + return; + } + String text = name.toUpperCase(Locale.ROOT); + int hash = 0; + for (int i = 0, n = text.length(); i < n; i++) { + hash = 31 * hash + text.charAt(i); + } + int slot = spread(hash) & MASK; + while (NAMES[slot] != null) { + if (NAMES[slot].equals(text)) { + return; + } + slot = (slot + 1) & MASK; + } + NAMES[slot] = text; + ZONES[slot] = zone; + } + + private static Zone find(CharSequence s, int lo, int hi, boolean isAsciiOnly) throws NumericException { + int hash = 0; + for (int i = lo; i < hi; i++) { + char c = s.charAt(i); + if (isAsciiOnly && c > 0x7f) { + throw NumericException.instance(); + } + hash = 31 * hash + Character.toUpperCase(c); + } + int length = hi - lo; + for (int slot = spread(hash) & MASK; NAMES[slot] != null; slot = (slot + 1) & MASK) { + String name = NAMES[slot]; + if (name.length() == length && regionMatches(s, lo, name)) { + return ZONES[slot]; + } + } + throw NumericException.instance(); + } + + private static boolean regionMatches(CharSequence value, int lo, String name) { + for (int i = 0, n = name.length(); i < n; i++) { + if (Character.toUpperCase(value.charAt(lo + i)) != name.charAt(i)) { + return false; + } + } + return true; + } + + private static int spread(int hash) { + return hash ^ (hash >>> 16); + } + } + + private static final class Zone { + // Seconds, or Long.MIN_VALUE when absent. + private final long cutoffSeconds; + private final long fixedOffsetSeconds; + private final long lastWallSeconds; + private final List recurring; + private final ZoneRules rules; + + private Zone(ZoneRules rules) { + this.rules = rules; + List history = rules.getTransitions(); + cutoffSeconds = history.isEmpty() ? Long.MIN_VALUE : history.get(history.size() - 1).toEpochSecond(); + recurring = rules.getTransitionRules(); + fixedOffsetSeconds = rules.isFixedOffset() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() : Long.MIN_VALUE; + lastWallSeconds = history.isEmpty() ? rules.getOffset(Instant.EPOCH).getTotalSeconds() + : history.get(history.size() - 1).getOffsetAfter().getTotalSeconds(); + } + + private static long transitionEpoch(Target t, ZoneOffsetTransitionRule rule, int year) { + boolean leap = CommonUtils.isLeapYear(year); + int month = rule.getMonth().getValue(); + int dom = rule.getDayOfMonthIndicator(); + int dow = rule.getDayOfWeek() == null ? -1 : rule.getDayOfWeek().getValue(); + long time = rule.getLocalTime().getHour() * t.hour + rule.getLocalTime().getMinute() * t.minute + + rule.getLocalTime().getSecond() * t.second; + long timestamp; + if (dom < 0) { + timestamp = yearStart(t, year, leap) + monthStart(t, month, leap) + + (CommonUtils.getDaysPerMonth(month, leap) + dom) * t.day + time; + if (dow > -1) { + timestamp = previousOrSame(t, timestamp, dow); + } + } else { + timestamp = yearStart(t, year, leap) + monthStart(t, month, leap) + (dom - 1L) * t.day + time; + if (dow > -1) { + timestamp = nextOrSame(t, timestamp, dow); + } + } + if (rule.isMidnightEndOfDay()) { + timestamp += t.day; + } + int before = rule.getOffsetBefore().getTotalSeconds(); + if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.UTC) { + timestamp += before * t.second; + } else if (rule.getTimeDefinition() == ZoneOffsetTransitionRule.TimeDefinition.STANDARD) { + timestamp += (before - rule.getStandardOffset().getTotalSeconds()) * t.second; + } + return timestamp - before * t.second; + } + + private long offset(Target t, long epoch, int year) throws NumericException { + if (fixedOffsetSeconds != Long.MIN_VALUE) { + return fixedOffsetSeconds * t.second; + } + long cutoff = cutoffSeconds == Long.MIN_VALUE ? Long.MIN_VALUE : cutoffSeconds * t.second; + if (!recurring.isEmpty() && epoch > cutoff) { + if (t.isNegativeYearRuleRejected && year < 0) { + // The server's DATE recurring-rule cache rejects negative + // year keys. This is only reachable after full-width DATE + // arithmetic wraps past the cutoff. + throw NumericException.instance(); + } + // The server computes future rules with the parsed year, + // even when calendar arithmetic has wrapped the epoch. + long after = 0; + for (int i = 0, n = recurring.size(); i < n; i++) { + ZoneOffsetTransitionRule rule = recurring.get(i); + if (epoch < transitionEpoch(t, rule, year)) { + return rule.getOffsetBefore().getTotalSeconds() * t.second; + } + after = rule.getOffsetAfter().getTotalSeconds() * t.second; + } + return after; + } + if (epoch > cutoff) { + return lastWallSeconds * t.second; + } + Instant instant = Instant.ofEpochSecond(Math.floorDiv(epoch, t.second), Math.floorMod(epoch, t.second) * t.nanos); + return rules.getOffset(instant).getTotalSeconds() * t.second; + } + } + } + + /** The server's TIMESTAMP (micros) or DATE (millis) text parser family. */ + private static final class Target { + private final long day; + private final long hour; + private final boolean isNegativeYearRuleRejected; + private final boolean isYearSaturating; + private final boolean isZoneAsciiOnly; + private final long milli; + private final long minute; + // Nanos per unit. + private final long nanos; + private final long second; + + private Target( + long unitsPerMilli, + boolean isYearSaturating, + boolean isZoneAsciiOnly, + boolean isNegativeYearRuleRejected + ) { + this.milli = unitsPerMilli; + this.second = 1_000L * unitsPerMilli; + this.minute = 60_000L * unitsPerMilli; + this.hour = 3_600_000L * unitsPerMilli; + this.day = 86_400_000L * unitsPerMilli; + this.nanos = 1_000_000L / unitsPerMilli; + this.isYearSaturating = isYearSaturating; + this.isZoneAsciiOnly = isZoneAsciiOnly; + this.isNegativeYearRuleRejected = isNegativeYearRuleRejected; + } + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java index 9f01f0ab8..c20c54bd3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpTableBuffer.java @@ -33,6 +33,7 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.CharSequenceIntHashMap; import io.questdb.client.std.Chars; +import io.questdb.client.std.Decimal; import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; @@ -63,6 +64,8 @@ public class QwpTableBuffer implements QuietCloseable { private static final int MAX_COLUMN_NAME_LENGTH = 127; private final LowerCaseCharSequenceIntHashMap columnNameToIndex; private final ObjList columns; + // lower-cased table name, the schema cache key; the name itself when already lower case + private final String schemaKey; private final QwpWebSocketSender sender; private final String tableName; private QwpColumnDef[] cachedColumnDefs; @@ -72,19 +75,22 @@ public class QwpTableBuffer implements QuietCloseable { private ColumnBuffer[] fastColumns; // plain array for O(1) sequential access private int inProgressColumnCount; private int rowCount; + private QwpSchemaBinding schemaBinding; public QwpTableBuffer(String tableName) { this(tableName, null); } /** - * Use this constructor overload to allow writing to a symbol column. - * {@link ColumnBuffer#addSymbol(CharSequence)} needs the sender to - * call {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}, registering - * the symbol in the global dictionary shared with the server. + * Use this constructor overload to encode symbol columns with the sender's + * global dictionary IDs. {@link ColumnBuffer#addSymbol(CharSequence)} calls + * {@link QwpWebSocketSender#getOrAddGlobalSymbol(CharSequence)}; buffers built + * without a sender instead maintain a local per-column symbol dictionary. */ public QwpTableBuffer(String tableName, QwpWebSocketSender sender) { this.tableName = tableName; + String lowerCaseName = Chars.toLowerCase(tableName); + this.schemaKey = tableName.equals(lowerCaseName) ? tableName : lowerCaseName; this.sender = sender; this.columns = new ObjList<>(); this.columnNameToIndex = new LowerCaseCharSequenceIntHashMap(); @@ -120,6 +126,7 @@ public void cancelCurrentRow() { * Frees all off-heap memory. */ public void clear() { + schemaBinding = null; for (int i = 0, n = columns.size(); i < n; i++) { columns.get(i).close(); } @@ -190,6 +197,42 @@ public ColumnBuffer getExistingColumn(CharSequence name, byte type) { return lookupColumn(name, type); } + ColumnBuffer getExistingColumnByName(CharSequence name) { + int index = columnNameToIndex.get(name); + return index >= 0 ? columns.get(index) : null; + } + + /** + * Claims {@code column}, found by {@link #findColumn}, for the in-progress + * row. Returns {@code null} when the row already holds a value for it, with + * the same first-value-wins semantics as {@link #getOrCreateColumn}. + */ + ColumnBuffer claimColumn(ColumnBuffer column) { + if (column.size > rowCount) { + return null; + } + inProgressColumnCount++; + return column; + } + + /** + * Finds a column by name without checking its type, advancing the + * sequential access cursor on a predicted hit exactly like + * {@link #getOrCreateColumn}. Returns {@code null} when absent. + */ + ColumnBuffer findColumn(CharSequence name) { + int n = columns.size(); + if (columnAccessCursor < n) { + ColumnBuffer candidate = fastColumns[columnAccessCursor]; + if (Chars.equalsIgnoreCase(candidate.name, name)) { + columnAccessCursor++; + return candidate; + } + } + int idx = columnNameToIndex.get(name); + return idx >= 0 ? columns.get(idx) : null; + } + /** * Gets or creates a column with the given name and type. *

@@ -211,11 +254,7 @@ public ColumnBuffer getOrCreateColumn(CharSequence name, byte type, boolean useN // col.size > rowCount means this column already received a value // for the in-progress row. Silently ignore the duplicate (first // value wins, same as the ILP server behaviour). - if (existing.size > rowCount) { - return null; - } - inProgressColumnCount++; - return existing; + return claimColumn(existing); } if (TableUtils.isValidColumnName(name, MAX_COLUMN_NAME_LENGTH)) { ColumnBuffer col = createColumn(name, type, useNullBitmap); @@ -256,6 +295,31 @@ public String getTableName() { return tableName; } + public QwpSchemaBinding getSchemaBinding() { + return schemaBinding; + } + + /** + * Returns the lower-cased table name that keys the schema cache, computed + * once so a schema lookup does not allocate. + */ + public String getSchemaKey() { + return schemaKey; + } + + void attachSchemaBinding(QwpSchemaBinding binding) { + if (schemaBinding != null || rowCount != 0 || hasInProgressRow() || columns.size() != 0) { + throw new IllegalStateException("schema binding requires an empty, unbound table buffer"); + } + schemaBinding = binding; + } + + void requireSchemaBinding(QwpSchemaBinding binding) { + if (schemaBinding != binding) { + throw new IllegalStateException("schema binding is no longer attached to the table buffer"); + } + } + /** * Returns true if a row is partially built (columns written but not yet committed * via {@link #nextRow()}). @@ -432,26 +496,11 @@ private ColumnBuffer createColumn(CharSequence name, byte type, boolean useNullB } private ColumnBuffer lookupColumn(CharSequence name, byte type) { - // Fast path: predict next column in sequence - int n = columns.size(); - if (columnAccessCursor < n) { - ColumnBuffer candidate = fastColumns[columnAccessCursor]; - if (Chars.equalsIgnoreCase(candidate.name, name)) { - columnAccessCursor++; - assertColumnType(name, type, candidate); - return candidate; - } - } - - // Slow path: hash map lookup - int idx = columnNameToIndex.get(name); - if (idx >= 0) { - ColumnBuffer existing = columns.get(idx); + ColumnBuffer existing = findColumn(name); + if (existing != null) { assertColumnType(name, type, existing); - return existing; } - - return null; + return existing; } private void rebuildColumnAccessStructures() { @@ -620,6 +669,7 @@ void reset(boolean forLong) { * operation and efficient bulk copy to network buffers. */ public static class ColumnBuffer implements QuietCloseable { + static final int SCHEMA_UNRESOLVED = Integer.MIN_VALUE; private static final long DOUBLE_ARRAY_BASE_OFFSET = Unsafe.getUnsafe().arrayBaseOffset(double[].class); final int elemSize; final String name; @@ -638,6 +688,11 @@ public static class ColumnBuffer implements QuietCloseable { private OffHeapAppendMemory dataBuffer; // Decimal storage private byte decimalScale = -1; + private boolean schemaDecimalScaleLocked; + // Schema column index the binding resolved this column to, -1 when the + // schema lacks it, or SCHEMA_UNRESOLVED before any binding resolved it. + private int schemaIndex = SCHEMA_UNRESOLVED; + private int inferredArrayDimensionality = -1; private double[] doubleArrayData; // GeoHash precision (number of bits, 1-60) private int geohashPrecision = -1; @@ -699,10 +754,17 @@ public void addBinary(byte[] value) { throw new LineSenderException( "BINARY value cannot be null; mark the row null via the null bitmap instead"); } - if (value.length > 0) { - stringData.putBytes(value, 0, value.length); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (value.length > 0) { + stringData.putBytes(value, 0, value.length); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; size++; } @@ -720,10 +782,17 @@ public void addBinary(long ptr, long len) { if (len > 0 && ptr == 0) { throw new LineSenderException("BINARY pointer cannot be 0 for a non-empty value"); } - if (len > 0) { - stringData.putBlockOfBytes(ptr, len); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (len > 0) { + stringData.putBlockOfBytes(ptr, len); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; size++; } @@ -827,6 +896,77 @@ public void addDecimal64(Decimal64 value) { size++; } + public boolean addSchemaDecimal(Decimal value, int targetPrecision, int targetScale) throws NumericException { + value.toDecimal256(rescaleTemp); + return appendSchemaDecimal(targetPrecision, targetScale); + } + + public void addSchemaDecimalNull(int targetScale) { + lockSchemaDecimalScale(targetScale); + addNull(); + } + + public boolean addSchemaLongDecimal(long value, int targetPrecision, int targetScale) throws NumericException { + rescaleTemp.ofLong(value, 0); + return appendSchemaDecimal(targetPrecision, targetScale); + } + + public boolean addSchemaStringDecimal(CharSequence value, int targetPrecision, int targetScale) throws NumericException { + rescaleTemp.ofString(value, 0, value.length(), targetPrecision, targetScale, false, false); + if (rescaleTemp.isNull()) { + addSchemaDecimalNull(targetScale); + return true; + } + return appendSchemaDecimal(targetPrecision, targetScale); + } + + private boolean appendSchemaDecimal(int targetPrecision, int targetScale) throws NumericException { + if (rescaleTemp.getScale() != targetScale) { + rescaleTemp.rescale(targetScale); + } + if (!rescaleTemp.comparePrecision(targetPrecision)) { + return false; + } + int storageSizePow2; + switch (type) { + case TYPE_DECIMAL64: + storageSizePow2 = 3; + break; + case TYPE_DECIMAL128: + storageSizePow2 = 4; + break; + case TYPE_DECIMAL256: + storageSizePow2 = 5; + break; + default: + throw new IllegalStateException("not a decimal column: " + type); + } + if (!rescaleTemp.fitsInStorageSizePow2(storageSizePow2)) { + return false; + } + lockSchemaDecimalScale(targetScale); + switch (type) { + case TYPE_DECIMAL64: + dataBuffer.putLong(rescaleTemp.getLl()); + break; + case TYPE_DECIMAL128: + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + break; + case TYPE_DECIMAL256: + dataBuffer.putLong(rescaleTemp.getHh()); + dataBuffer.putLong(rescaleTemp.getHl()); + dataBuffer.putLong(rescaleTemp.getLh()); + dataBuffer.putLong(rescaleTemp.getLl()); + break; + default: + throw new AssertionError(); + } + valueCount++; + size++; + return true; + } + public void addDouble(double value) { dataBuffer.putDouble(value); valueCount++; @@ -946,6 +1086,13 @@ public void addFloat(float value) { * @param precision number of bits (1-60) */ public void addGeoHash(long value, int precision) { + initGeoHashPrecision(precision); + dataBuffer.putLong(value); + valueCount++; + size++; + } + + public void initGeoHashPrecision(int precision) { if (precision < 1 || precision > 60) { throw new LineSenderException("invalid GeoHash precision: " + precision + " (must be 1-60)"); } @@ -956,9 +1103,6 @@ public void addGeoHash(long value, int precision) { "GeoHash precision mismatch: column has " + geohashPrecision + " bits, got " + precision ); } - dataBuffer.putLong(value); - valueCount++; - size++; } public void addInt(int value) { @@ -1180,10 +1324,17 @@ public void addString(CharSequence value) { ensureNullBitmapCapacity(size + 1); markNull(size); } else { - if (value != null) { - stringData.putUtf8(value); + long dataOffset = stringData.getAppendOffset(); + long offsetsOffset = stringOffsets.getAppendOffset(); + try { + if (value != null) { + stringData.putUtf8(value); + } + stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); + } catch (RuntimeException | Error e) { + restoreStringAppendPositions(dataOffset, offsetsOffset); + throw e; } - stringOffsets.putInt(checkedStringOffset(stringData.getAppendOffset())); valueCount++; } size++; @@ -1373,7 +1524,11 @@ public long getDataAddress() { } public byte getDecimalScale() { - return decimalScale; + // A decimal column containing only nulls has no natural scale yet. + // Scale -1 is internal state, not valid QWP metadata; emit the + // deterministic scale-zero representation without locking it so a + // later finite batch can still select its natural scale. + return decimalScale == -1 ? 0 : decimalScale; } public double[] getDoubleArrayData() { @@ -1404,10 +1559,25 @@ public long getNullBitmapAddress() { return nullBufPtr; } + int getSchemaIndex() { + return schemaIndex; + } + public int getSize() { return size; } + void setSchemaIndex(int schemaIndex) { + this.schemaIndex = schemaIndex; + } + + int pinInferredArrayDimensionality(int dimensionality) { + if (inferredArrayDimensionality < 0) { + inferredArrayDimensionality = dimensionality; + } + return inferredArrayDimensionality; + } + public long getStringDataAddress() { return stringData != null ? stringData.pageAddress() : 0; } @@ -1454,6 +1624,10 @@ public boolean hasSymbol(CharSequence value) { return symbolDict != null && symbolDict.get(value) != CharSequenceIntHashMap.NO_ENTRY_VALUE; } + boolean isSchemaResolved() { + return schemaIndex != SCHEMA_UNRESOLVED; + } + public boolean isNull(int index) { if (nullBufPtr == 0 || index >= nullBufCapRows) { return false; @@ -1491,7 +1665,9 @@ public void reset() { maxGlobalSymbolId = -1; arrayShapeOffset = 0; arrayDataOffset = 0; - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } // geohashPrecision is intentionally NOT cleared here. It is a // schema property, locked on first write and matched by the // server's auto-created GEOHASH(Nb) type, so preserving it across @@ -1610,7 +1786,9 @@ public void truncateTo(int newSize) { // through ColumnBuffer.reset() and intentionally preserves // geohashPrecision there to keep its server-locked value. if (newValueCount == 0) { - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } geohashPrecision = -1; maxGlobalSymbolId = -1; storeGlobalSymbolIdsOnly = false; @@ -1625,6 +1803,11 @@ public boolean usesNullBitmap() { return useNullBitmap; } + private void restoreStringAppendPositions(long dataOffset, long offsetsOffset) { + stringData.jumpTo(dataOffset); + stringOffsets.jumpTo(offsetsOffset); + } + private static int checkedElementCount(long product) { if (product > Integer.MAX_VALUE) { throw new LineSenderException("array too large: total element count exceeds int range"); @@ -1852,8 +2035,21 @@ private void markNull(int index) { hasNulls = true; } + private void lockSchemaDecimalScale(int targetScale) { + if (schemaDecimalScaleLocked) { + if ((decimalScale & 0xFF) != targetScale) { + throw new IllegalStateException("schema decimal scale changed"); + } + return; + } + decimalScale = (byte) targetScale; + schemaDecimalScaleLocked = true; + } + private void resetEmptyMetadata() { - decimalScale = -1; + if (!schemaDecimalScaleLocked) { + decimalScale = -1; + } geohashPrecision = -1; maxGlobalSymbolId = -1; storeGlobalSymbolIdsOnly = false; diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index c9529a13e..613694d38 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -83,6 +83,8 @@ public final class ConfigSchema { str("reconnect_max_backoff_millis", Side.INGRESS); str("reconnect_max_duration_millis", Side.INGRESS); str("request_durable_ack", Side.INGRESS); + str("schema_mode", Side.INGRESS); + str("schema_wait_millis", Side.INGRESS); str("sender_id", Side.INGRESS); str("sf_append_deadline_millis", Side.INGRESS); str("sf_dir", Side.INGRESS); diff --git a/core/src/main/java/io/questdb/client/std/DecimalParser.java b/core/src/main/java/io/questdb/client/std/DecimalParser.java index 86fabdcbc..e3d8aee14 100644 --- a/core/src/main/java/io/questdb/client/std/DecimalParser.java +++ b/core/src/main/java/io/questdb/client/std/DecimalParser.java @@ -35,7 +35,7 @@ *

  • QuestDB decimal suffix: "123.45m" or "123.45M"
    • *
  • Floating-point suffixes: "123.45f", "123.45d" (stripped during parsing)
    • *
  • Leading zeros: "00123.45" → 123.45
    • - *
  • Leading/trailing whitespace
    • + *
  • Leading ASCII spaces
    • * * * @see Decimal @@ -54,7 +54,7 @@ public final class DecimalParser { *

    * Parsing Process: *

      - *
    1. Strip suffixes (m, M, f, d) and leading/trailing whitespace
      2. + *
    2. Strip supported suffixes (m, M, f, d) and leading ASCII spaces
      3. *
    3. Parse sign (+/-) and detect special values (NaN/Infinity)
      4. *
    4. Remove leading zeros
      5. *
    5. Extract mantissa and locate decimal point
      6. @@ -163,12 +163,21 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // We do a first pass over the literal to ensure that the format is correct (numerical and at most 1 dot) and to // measure the given precision/scale. int dot = -1; + int leadingZeroes = 0; boolean digitFound = false; + boolean hasSignificantDigit = false; int digitLo = lo; for (; lo < hi; lo++) { char c = cs.charAt(lo); if (isDigit(c)) { digitFound = true; + if (!hasSignificantDigit) { + if (c == '0') { + leadingZeroes++; + } else { + hasSignificantDigit = true; + } + } continue; } else if (c == '.' && dot == -1) { dot = lo; @@ -178,7 +187,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p } if (!digitFound) { if (skippedZeroes) { + // The stripper only walks over zeroes, so the digit it gives back is a zero. digitLo--; + leadingZeroes = 1; } else { throw NumericException.instance() .put("invalid decimal: '").put(cs) @@ -196,9 +207,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p int literalScale = dot == -1 ? 0 : (digitHi - dot - 1); int literalDigits = digitHi - digitLo - (dot == -1 ? 0 : 1); - int virtualScale = literalScale; + long virtualScale = literalScale; - int exp = 0; + long exp = 0; if (lo != hi) { // Parses exponent if ((cs.charAt(lo) | 32) == 'e') { @@ -222,7 +233,7 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // If lossy is enabled, we can strip digits after the provided scale (as long as it is provided) if (lossy && scale >= 0 && virtualScale > scale) { - int drop = virtualScale - scale; + long drop = virtualScale - scale; while (drop > 0 && digitHi > digitLo) { digitHi--; if (digitHi == dot) { @@ -237,7 +248,12 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // We still need to adjust the exponent if the user specifies a target scale. if (scale == -1) { - scale = virtualScale; + if (virtualScale > decimal.getMaxScale()) { + throw NumericException.instance() + .put("decimal '").put(cs) + .put("' exceeds maximum allowed scale of ").put(decimal.getMaxScale()); + } + scale = (int) virtualScale; } else if (virtualScale <= scale) { exp += scale - virtualScale; } else { @@ -258,8 +274,9 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p // precision, as long as it's lower. // Compute the final precision of the decimal - int pow = literalDigits + exp; - final int finalPrecision = Math.max(pow, scale + 1); + long pow = literalDigits + exp; + final long unscaledDigits = literalDigits > leadingZeroes ? pow - leadingZeroes : 0; + final long finalPrecision = Math.max(Math.max(unscaledDigits, scale), 1); if (precision != -1 && finalPrecision > precision) { throw NumericException.instance() .put("decimal '").put(cs) @@ -277,8 +294,14 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p if (p == dot) { continue; } - - decimal.addPowerOfTenMultiple(--pow, cs.charAt(p) - '0'); + int multiplier = cs.charAt(p) - '0'; + pow--; + if (multiplier != 0) { + if (pow < 0 || pow >= decimal.getMaxPrecision()) { + throw NumericException.instance().put("decimal '").put(cs).put("' exponent is out of range"); + } + decimal.addPowerOfTenMultiple((int) pow, multiplier); + } } decimal.setScale(scale); @@ -286,7 +309,7 @@ public static long parse(Decimal decimal, CharSequence cs, int lo, int hi, int p decimal.negate(); } - return Numbers.encodeLowHighInts(finalPrecision, scale); + return Numbers.encodeLowHighInts((int) finalPrecision, scale); } /** @@ -315,9 +338,11 @@ private static boolean isDigit(char c) { * @return true if "NaN" or "Infinity" is found at position lo, false otherwise */ private static boolean isNanOrInfinite(char ch, CharSequence cs, int lo, int hi) { + // A prefix match, like the server's DecimalParser: "NaNjunk" parses as NULL on + // every server ingest path, so the client must not reject it. return (ch == 'N' && lo + 2 < hi && cs.charAt(lo + 1) == 'a' && cs.charAt(lo + 2) == 'N') || (ch == 'I' && lo + 7 < hi && cs.charAt(lo + 1) == 'n' && cs.charAt(lo + 2) == 'f' && cs.charAt(lo + 3) == 'i' && cs.charAt(lo + 4) == 'n' && cs.charAt(lo + 5) == 'i' && cs.charAt(lo + 6) == 't' && cs.charAt(lo + 7) == 'y'); } -} \ No newline at end of file +} diff --git a/core/src/main/java/io/questdb/client/std/Numbers.java b/core/src/main/java/io/questdb/client/std/Numbers.java index 78a3bf4c7..6bda0ace7 100644 --- a/core/src/main/java/io/questdb/client/std/Numbers.java +++ b/core/src/main/java/io/questdb/client/std/Numbers.java @@ -70,7 +70,10 @@ public final class Numbers { private static final int[] insignificantDigitsNumber = new int[]{0, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19}; private static final LongHexAppender[] longHexAppender = new LongHexAppender[Long.SIZE + 1]; private static final LongHexAppender[] longHexAppenderPad64 = new LongHexAppender[Long.SIZE + 1]; - private final static ThreadLocal tlDoubleDigitsBuffer = new ThreadLocal<>(() -> new char[21]); + // Scratch cell for RyuDouble.d2d()'s out-parameter. The server passes this + // through CharSink.ryuScratch(); the client's CharSink has no such method, + // so the single shared formatter keeps its own per-thread cell instead. + private final static ThreadLocal tlRyuScratch = new ThreadLocal<>(() -> new int[1]); private Numbers() { } @@ -175,47 +178,92 @@ public static void append(CharSink sink, double value) { } public static void append(CharSink sink, double value, int scale) { - final char[] digits = tlDoubleDigitsBuffer.get(); final long doubleBits = Double.doubleToRawLongBits(value); - boolean negative = (doubleBits & SIGN_BIT_MASK) != 0L; - long significantBitCount = doubleBits & SIGNIF_BIT_MASK; - int binExp = (int) ((doubleBits & EXP_BIT_MASK) >> EXP_SHIFT); - - if (binExp == 2047) { - if (significantBitCount == 0L) { - if (negative) { - sink.putAscii("-Infinity"); - } else { - sink.putAscii("Infinity"); + long ieeeMantissa = doubleBits & SIGNIF_BIT_MASK; + int ieeeExponent = (int) ((doubleBits & EXP_BIT_MASK) >> EXP_SHIFT); + + // NaN or Infinity + if (ieeeExponent == 2047) { + if (ieeeMantissa != 0) { + sink.putAscii("NaN"); + } else if ((doubleBits & SIGN_BIT_MASK) != 0) { + sink.putAscii("-Infinity"); + } else { + sink.putAscii("Infinity"); + } + return; + } + + // Negative sign (including -0.0) + if ((doubleBits & SIGN_BIT_MASK) != 0) { + sink.putAscii('-'); + } + + // Zero + if (ieeeExponent == 0 && ieeeMantissa == 0) { + sink.putAscii("0.0"); + return; + } + + // Decompose via Ryu + int[] e10 = tlRyuScratch.get(); + long output = RyuDouble.d2d(ieeeMantissa, ieeeExponent, e10); + int olength = RyuDouble.decimalLength17(output); + int decExp = e10[0] + olength; + + if (decExp > 0 && decExp < 8) { + // Fixed-point with integer part: e.g. "1234.567" or "1234000.0" + if (olength <= decExp) { + for (int i = 0; i < olength; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); + } + for (int i = olength; i < decExp; i++) { + sink.putAscii('0'); } + sink.putAscii(".0"); } else { - sink.putAscii("NaN"); + int fracDigits = olength - decExp; + if (scale < MAX_DOUBLE_SCALE && fracDigits > scale) { + fracDigits = Math.max(scale, 1); + } + for (int i = 0; i < decExp; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); + } + sink.putAscii('.'); + for (int i = 0; i < fracDigits; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - decExp - i] % 10))); + } + } + } else if (decExp <= 0 && decExp > -3) { + // Leading-zero fixed-point: e.g. "0.00123" + int leadingZeros = -decExp; + int totalFrac = leadingZeros + olength; + int digitsFromOutput = olength; + if (scale < MAX_DOUBLE_SCALE && totalFrac > scale) { + totalFrac = Math.max(scale, 1); + digitsFromOutput = Math.max(totalFrac - leadingZeros, 0); + } + int zerosToWrite = Math.min(leadingZeros, totalFrac); + sink.putAscii("0."); + for (int i = 0; i < zerosToWrite; i++) { + sink.putAscii('0'); + } + for (int i = 0; i < digitsFromOutput; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); } } else { - int fractionBits; - if (binExp == 0) { - if (significantBitCount == 0L) { - if (negative) { - sink.putAscii("-0.0"); - } else { - sink.putAscii("0.0"); - } - return; + // Scientific notation: e.g. "1.23E8" or "1.0E-4" + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1] % 10))); + sink.putAscii('.'); + if (olength > 1) { + for (int i = 1; i < olength; i++) { + sink.putAscii((char) ('0' + (int) (output / pow10[olength - 1 - i] % 10))); } - - int leadingZeros = Long.numberOfLeadingZeros(significantBitCount); - int shift = leadingZeros - (63 - EXP_SHIFT); - significantBitCount <<= shift; - binExp = 1 - shift; - fractionBits = 64 - leadingZeros; } else { - significantBitCount |= FRACT_HOB; - fractionBits = 53; + sink.putAscii('0'); } - - binExp -= EXP_BIAS; - - appendDouble0(binExp, significantBitCount, fractionBits, negative, digits, sink, scale); + sink.putAscii('E'); + append(sink, decExp - 1); } } @@ -406,7 +454,10 @@ public static int parseIPv4_0(CharSequence sequence, final int p, int lim) throw if (sign == '.') { do { lo++; - } while (sequence.charAt(lo) == '.'); + } while (lo < lim && sequence.charAt(lo) == '.'); + if (lo == lim) { + throw NumericException.instance().put("IPv4 address must have 4 octets, found: 1"); + } } else { throw NumericException.instance().put("invalid IPv4 address: ").put(sequence); } @@ -477,347 +528,6 @@ public static long parseLong(Utf8Sequence sequence) throws NumericException { return parseLong0(sequence.asAsciiCharSequence(), sequence.size()); } - private static void appendDouble0( - int binExp, - long fractionBits, - int significantBitCount, - boolean negative, - char[] digits, - CharSink out, - int outScale - ) { - assert fractionBits > 0L; - assert (fractionBits & FRACT_HOB) != 0L; - - final int tailZeroes = Long.numberOfTrailingZeros(fractionBits); - final int fractBitCount = EXP_SHIFT + 1 - tailZeroes; - int decExp; - int firstDigitIndex; - int nDigits; - - final int tinyBitCount = Math.max(0, fractBitCount - binExp - 1); - if (binExp < MAX_SMALL_BIN_EXP + 1 && binExp > MIN_SMALL_BIN_EXP - 1 && tinyBitCount < LONG_5_POW.length && fractBitCount + N_5_BITS[tinyBitCount] < 64 && tinyBitCount == 0) { - int insignificant; - if (binExp > significantBitCount) { - insignificant = insignificantDigitsForPow2(binExp - significantBitCount - 1); - } else { - insignificant = 0; - } - - if (binExp >= EXP_SHIFT) { - fractionBits <<= binExp - EXP_SHIFT; - } else { - fractionBits >>>= EXP_SHIFT - binExp; - } - - // - int binExp2 = 0; - if (insignificant != 0) { - long pow10 = LONG_5_POW[insignificant] << insignificant; - long residue = fractionBits % pow10; - fractionBits /= pow10; - binExp2 += insignificant; - if (residue >= pow10 >> 1) { - ++fractionBits; - } - } - - int digitIndex = digits.length - 1; - int digit; - if (fractionBits <= Integer.MAX_VALUE) { - assert fractionBits > 0L : fractionBits; - - int fractRemaining = (int) fractionBits; - digit = fractRemaining % 10; - - for (fractRemaining /= 10; digit == 0; fractRemaining /= 10) { - ++binExp2; - digit = fractRemaining % 10; - } - - while (fractRemaining != 0) { - digits[digitIndex--] = (char) (digit + '0'); - ++binExp2; - digit = fractRemaining % 10; - fractRemaining /= 10; - } - - } else { - digit = (int) (fractionBits % 10L); - - for (fractionBits /= 10L; digit == 0; fractionBits /= 10L) { - ++binExp2; - digit = (int) (fractionBits % 10L); - } - - while (fractionBits != 0L) { - digits[digitIndex--] = (char) (digit + '0'); - ++binExp2; - digit = (int) (fractionBits % 10L); - fractionBits /= 10L; - } - - } - digits[digitIndex] = (char) (digit + '0'); - - decExp = binExp2 + 1; - firstDigitIndex = digitIndex; - nDigits = digits.length - digitIndex; - } else { - int estDecExp = estimateDecExpDouble(fractionBits, binExp); - int B5 = Math.max(0, -estDecExp); - int B2 = B5 + tinyBitCount + binExp; - int S5 = Math.max(0, estDecExp); - int S2 = S5 + tinyBitCount; - int M2 = B2 - significantBitCount; - fractionBits >>>= tailZeroes; - B2 -= fractBitCount - 1; - int common2factor = Math.min(B2, S2); - B2 -= common2factor; - S2 -= common2factor; - M2 -= common2factor; - if (fractBitCount == 1) { - --M2; - } - - if (M2 < 0) { - B2 -= M2; - S2 -= M2; - M2 = 0; - } - - int bBits = fractBitCount + B2 + (B5 < N_5_BITS.length ? N_5_BITS[B5] : B5 * 3); - int tenBits = S2 + 1 + (S5 + 1 < N_5_BITS.length ? N_5_BITS[S5 + 1] : (S5 + 1) * 3); - boolean low; - boolean high; - long lowDigitDifference; - int q; - int digitIndex; - if (bBits < 64 && tenBits < 64) { - if (bBits < 32 && tenBits < 32) { - int b = (int) fractionBits * SMALL_5_POW[B5] << B2; - int s = SMALL_5_POW[S5] << S2; - int m = SMALL_5_POW[B5] << M2; - int tens = s * 10; - digitIndex = 0; - q = b / s; - b = 10 * (b % s); - m *= 10; - low = b < m; - high = b + m > tens; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - for (; !low && !high; digits[digitIndex++] = (char) ('0' + q)) { - q = b / s; - b = 10 * (b % s); - m *= 10; - - assert q < 10 : q; - - if ((long) m > 0L) { - low = b < m; - high = b + m > tens; - } else { - low = true; - high = true; - } - } - - lowDigitDifference = ((long) b << 1) - tens; - } else { - long b = fractionBits * LONG_5_POW[B5] << B2; - long s = LONG_5_POW[S5] << S2; - long m = LONG_5_POW[B5] << M2; - long tens = s * 10L; - digitIndex = 0; - q = (int) (b / s); - b = 10L * (b % s); - m *= 10L; - low = b < m; - high = b + m > tens; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - for (; !low && !high; digits[digitIndex++] = (char) ('0' + q)) { - q = (int) (b / s); - b = 10L * (b % s); - m *= 10L; - - assert q < 10 : q; - - if (m > 0L) { - low = b < m; - high = b + m > tens; - } else { - low = true; - high = true; - } - } - lowDigitDifference = (b << 1) - tens; - } - } else { - FdBig sVal = FdBig.valueOfPow52(S5, S2); - final int shiftBias = sVal.getNormalizationBias(); - sVal = sVal.leftShift(shiftBias); - FdBig bVal = FdBig.valueOfMulPow52(fractionBits, B5, B2 + shiftBias); - FdBig mVal = FdBig.valueOfPow52(B5 + 1, M2 + shiftBias + 1); - FdBig tensVal = FdBig.valueOfPow52(S5 + 1, S2 + shiftBias + 1); - digitIndex = 0; - q = bVal.quoRemIteration(sVal); - low = bVal.cmp(mVal) < 0; - high = tensVal.addAndCmp(bVal, mVal) <= 0; - - assert q < 10 : q; - - if (q == 0 && !high) { - --estDecExp; - } else { - digits[digitIndex++] = (char) ('0' + q); - } - - if (estDecExp < -3 || estDecExp >= 8) { - low = false; - high = false; - } - - while (!low && !high) { - q = bVal.quoRemIteration(sVal); - - assert q < 10 : q; - - mVal = mVal.multBy10(); - low = bVal.cmp(mVal) < 0; - high = tensVal.addAndCmp(bVal, mVal) <= 0; - digits[digitIndex++] = (char) ('0' + q); - } - - if (high && low) { - bVal = bVal.leftShift(1); - lowDigitDifference = bVal.cmp(tensVal); - } else { - lowDigitDifference = 0L; - } - } - - decExp = estDecExp + 1; - firstDigitIndex = 0; - nDigits = digitIndex; - if (high) { - if (low) { - if (lowDigitDifference == 0L) { - if ((digits[firstDigitIndex + nDigits - 1] & 1) != 0) { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } else if (lowDigitDifference > 0L) { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } else { - if (roundupDouble(firstDigitIndex, digits, nDigits)) { - decExp++; - } - } - } - } - - appendDouble00(digits, firstDigitIndex, nDigits, negative, decExp, out, outScale); - } - - private static void appendDouble00( - char[] digits, - int firstDigitIndex, - int nDigits, - boolean isNegative, - int decExp, - CharSink sink, - int outScale - ) { - assert nDigits <= MAX_DOUBLE_SCALE : nDigits; - if (isNegative) { - sink.putAscii('-'); - } - - int exp; - if (decExp > 0 && decExp < 8) { - exp = Math.min(nDigits, decExp); - sink.putAscii(digits, firstDigitIndex, exp); - if (exp < decExp) { - exp = decExp - exp; - sink.fillAscii('0', exp); - sink.putAscii('.'); - sink.putAscii('0'); - } else { - sink.putAscii('.'); - if (exp < nDigits) { - sink.putAscii(digits, firstDigitIndex + exp, Math.min(nDigits - exp, outScale)); - } else { - sink.putAscii('0'); - } - } - } else if (decExp <= 0 && decExp > -3) { - sink.putAscii('0').putAscii('.'); - if (decExp != 0) { - sink.fillAscii('0', -decExp); - } - - sink.putAscii(digits, firstDigitIndex, Math.min(nDigits, outScale)); - } else { - sink.putAscii(digits[firstDigitIndex]); - sink.putAscii('.'); - if (nDigits > 1) { - sink.putAscii(digits, firstDigitIndex + 1, nDigits - 1); - } else { - sink.putAscii('0'); - } - - sink.putAscii('E'); - if (decExp <= 0) { - sink.putAscii('-'); - exp = -decExp + 1; - } else { - exp = decExp - 1; - } - - if (exp < 10) { - sink.putAscii((char) (exp + '0')); - } else if (exp < 100) { - sink.putAscii((char) (exp / 10 + '0')); - sink.putAscii((char) (exp % 10 + '0')); - } else { - sink.putAscii((char) (exp / 100 + '0')); - exp %= 100; - sink.putAscii((char) (exp / 10 + '0')); - sink.putAscii((char) (exp % 10 + '0')); - } - } - } - private static void appendInt10(CharSink sink, int i) { int c; sink.putAscii((char) ('0' + i / 1000000000)); diff --git a/core/src/main/java/io/questdb/client/std/RyuDouble.java b/core/src/main/java/io/questdb/client/std/RyuDouble.java new file mode 100644 index 000000000..e20fb56f7 --- /dev/null +++ b/core/src/main/java/io/questdb/client/std/RyuDouble.java @@ -0,0 +1,637 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.std; + +/** + * Port of the QuestDB server's {@code io.questdb.std.RyuDouble}, kept + * byte-compatible with it so that client-side and server-side formatting of + * the same double produce identical text. + * Ryu algorithm for converting IEEE 754 double-precision floating-point numbers + * to their shortest decimal representation. + *

      + * Based on the paper "Ryu: Fast Float-to-String Conversion" by Ulf Adams (2018). + * This implementation uses 128-bit arithmetic via {@link Compat#multiplyHigh}, + * which is {@code Math.multiplyHigh} on Java 9+ and a hand-rolled equivalent on Java 8. + *

      + * The core method {@link #d2d} returns the shortest decimal significand as a {@code long} + * (1-17 digits) and writes the decimal exponent to a caller-provided array. The result + * satisfies: {@code value = output * 10^e10}. + */ +class RyuDouble { + private static final int DOUBLE_POW5_BITCOUNT = 125; + private static final int DOUBLE_POW5_INV_BITCOUNT = 125; + // ceil(2^(pow5bits(q) - 1 + 125) / 5^q) for q in [0, 291], stored as (hi64, lo64) pairs. + private static final long[] DOUBLE_POW5_INV_SPLIT = initPow5InvSplit(); + // Upper 128 bits of 5^i for i in [0, 325], stored as (hi64, lo64) pairs. + private static final long[] DOUBLE_POW5_SPLIT = initPow5Split(); + + private static long[] initPow5InvSplit() { + return new long[]{ + 0x2000000000000000L, 0x0000000000000001L, 0x1999999999999999L, 0x999999999999999AL, + 0x147AE147AE147AE1L, 0x47AE147AE147AE15L, 0x10624DD2F1A9FBE7L, 0x6C8B4395810624DEL, + 0x1A36E2EB1C432CA5L, 0x7A786C226809D496L, 0x14F8B588E368F084L, 0x61F9F01B866E43ABL, + 0x10C6F7A0B5ED8D36L, 0xB4C7F34938583622L, 0x1AD7F29ABCAF4857L, 0x87A6520EC08D236AL, + 0x15798EE2308C39DFL, 0x9FB841A566D74F88L, 0x112E0BE826D694B2L, 0xE62D01511F12A607L, + 0x1B7CDFD9D7BDBAB7L, 0xD6AE6881CB5109A4L, 0x15FD7FE17964955FL, 0xDEF1ED34A2A73AEAL, + 0x119799812DEA1119L, 0x7F27F0F6E885C8BBL, 0x1C25C268497681C2L, 0x650CB4BE40D60DF8L, + 0x16849B86A12B9B01L, 0xEA70909833DE7193L, 0x1203AF9EE756159BL, 0x21F3A6E0297EC143L, + 0x1CD2B297D889BC2BL, 0x6985D7CD0F313537L, 0x170EF54646D49689L, 0x2137DFD73F5A90F9L, + 0x12725DD1D243ABA0L, 0xE75FE645CC4873FAL, 0x1D83C94FB6D2AC34L, 0xA5663D3C7A0D865DL, + 0x179CA10C9242235DL, 0x511E976394D79EB1L, 0x12E3B40A0E9B4F7DL, 0xDA7EDF82DD794BC1L, + 0x1E392010175EE596L, 0x2A6498D1625BAC68L, 0x182DB34012B25144L, 0xEEB6E0A781E2F053L, + 0x1357C299A88EA76AL, 0x58924D52CE4F26A9L, 0x1EF2D0F5DA7DD8AAL, 0x27507BB7B07EA441L, + 0x18C240C4AECB13BBL, 0x52A6C95FC0655034L, 0x13CE9A36F23C0FC9L, 0x0EEBD44C99EAA690L, + 0x1FB0F6BE50601941L, 0xB17953ADC3110A80L, 0x195A5EFEA6B34767L, 0xC12DDC8B02740867L, + 0x14484BFEEBC29F86L, 0x3424B06F3529A052L, 0x1039D66589687F9EL, 0x901D59F290EE19DBL, + 0x19F623D5A8A73297L, 0x4CFBC31DB4B0295FL, 0x14C4E977BA1F5BACL, 0x3D9635B15D59BAB2L, + 0x109D8792FB4C4956L, 0x97AB5E277DE16228L, 0x1A95A5B7F87A0EF0L, 0xF2ABC9D8C9689D0DL, + 0x154484932D2E725AL, 0x5BBCA17A3ABA173EL, 0x11039D428A8B8EAEL, 0xAFCA1AC82EFB45CBL, + 0x1B38FB9DAA78E44AL, 0xB2DCF7A6B1920945L, 0x15C72FB1552D836EL, 0xF57D92EBC141A104L, + 0x116C262777579C58L, 0xC46475896767B403L, 0x1BE03D0BF225C6F4L, 0x6D6D88DBD8A5ECD2L, + 0x164CFDA3281E38C3L, 0x8ABE071646EB23DBL, 0x11D7314F534B609CL, 0x6EFE6C11D255B649L, + 0x1C8B821885456760L, 0xB197134FB6EF8A0EL, 0x16D601AD376AB91AL, 0x27AC0F72F8BFA1A5L, + 0x1244CE242C5560E1L, 0xB95672C260994E1EL, 0x1D3AE36D13BBCE35L, 0xF5571E03CDC21695L, + 0x17624F8A762FD82BL, 0x2AAC18030B01ABABL, 0x12B50C6EC4F31355L, 0xBBBCE0026F348956L, + 0x1DEE7A4AD4B81EEFL, 0x92C7CCD0B1EDA889L, 0x17F1FB6F10934BF2L, 0xDBD30A408E57BA07L, + 0x1327FC58DA0F6FF5L, 0x7CA8D50071DFC806L, 0x1EA6608E29B24CBBL, 0xFAA7BB33E9660CD6L, + 0x18851A0B548EA3C9L, 0x9552FC298784D711L, 0x139DAE6F76D88307L, 0xAAA8C9BAD2D0AC0EL, + 0x1F62B0B257C0D1A5L, 0xDDDADC5E1E1AACE3L, 0x191BC08EAC9A4151L, 0x7E48B04B4B488A4FL, + 0x141633A556E1CDDAL, 0xCB6D59D5D5D3A1D9L, 0x1011C2EAABE7D7E2L, 0x3C577B1177DC817BL, + 0x19B604AAACA62636L, 0xC6F25E825960CF2AL, 0x14919D5556EB51C5L, 0x6BF518684780A5BBL, + 0x10747DDDDF22A7D1L, 0x232A79ED06008496L, 0x1A53FC9631D10C81L, 0xD1DD8FE1A3340756L, + 0x150FFD44F4A73D34L, 0xA7E4731AE8F66C45L, 0x10D9976A5D52975DL, 0x531D28E253F8569EL, + 0x1AF5BF109550F22EL, 0xEB61DB03B98D5762L, 0x159165A6DDDA5B58L, 0xBC4E48CFC7A445E8L, + 0x11411E1F17E1E2ADL, 0x6371D3D96C836B20L, 0x1B9B6364F3030448L, 0x9F1C8628AD9F11CDL, + 0x1615E91D8F359D06L, 0xE5B06B53BE18DB0BL, 0x11AB20E472914A6BL, 0xEAF3890FCB4715A2L, + 0x1C45016D841BAA46L, 0x44B8DB4C7871BC37L, 0x169D9ABE03495505L, 0x03C715D6C6C1635FL, + 0x1217AEFE69077737L, 0x3638DE456BCDE919L, 0x1CF2B1970E725858L, 0x56C163A2461641C1L, + 0x17288E1271F51379L, 0xDF011C81D1AB67CEL, 0x1286D80EC190DC61L, 0x7F3416CE4155ECA5L, + 0x1DA48CE468E7C702L, 0x6520247D3556476EL, 0x17B6D71D20B96C01L, 0xEA801D30F7783925L, + 0x12F8AC174D612334L, 0xBB99B0F3F92CFA84L, 0x1E5AACF215683854L, 0x5F5C4E532847F739L, + 0x18488A5B44536043L, 0x7F7D0B75B9D32C2EL, 0x136D3B7C36A919CFL, 0x9930D5F7C7DC2358L, + 0x1F152BF9F10E8FB2L, 0x8EB4898C72F9D226L, 0x18DDBCC7F40BA628L, 0x722A07A38F2E41B8L, + 0x13E497065CD61E86L, 0xC1BB394FA5BE9AFAL, 0x1FD424D6FAF030D7L, 0x9C5EC2190930F7F6L, + 0x197683DF2F268D79L, 0x49E56814075A5FF8L, 0x145ECFE5BF520AC7L, 0x6E51201005E1E660L, + 0x104BD984990E6F05L, 0xF1DA800CD181851AL, 0x1A12F5A0F4E3E4D6L, 0x4FC400148268D4F5L, + 0x14DBF7B3F71CB711L, 0xD96999AA01ED772BL, 0x10AFF95CC5B09274L, 0xADEE1488018AC5BCL, + 0x1AB328946F80EA54L, 0x497CEDA668DE092CL, 0x155C2076BF9A5510L, 0x3ACA57B853E4D424L, + 0x1116805EFFAEAA73L, 0x623B7960431D7683L, 0x1B5733CB32B110B8L, 0x9D2BF566D1C8BD9EL, + 0x15DF5CA28EF40D60L, 0x7DBCC452416D647FL, 0x117F7D4ED8C33DE6L, 0xCAFD69DB678AB6CCL, + 0x1BFF2EE48E052FD7L, 0xAB2F0FC572778ADFL, 0x1665BF1D3E6A8CACL, 0x88F273045B92D580L, + 0x11EAFF4A98553D56L, 0xD3F528D049424466L, 0x1CAB3210F3BB9557L, 0xB988414D4203A0A3L, + 0x16EF5B40C2FC7779L, 0x6139CDD76802E6E9L, 0x125915CD68C9F92DL, 0xE761717920025254L, + 0x1D5B561574765B7CL, 0xA568B58E999D5086L, 0x177C44DDF6C515FDL, 0x5120913EE14AA6D2L, + 0x12C9D0B1923744CAL, 0xA74D40FF1AA21F0EL, 0x1E0FB44F50586E11L, 0x0BAECE64F769CB4AL, + 0x180C903F7379F1A7L, 0x3C8BD850C5EE3C3BL, 0x133D4032C2C7F485L, 0xCA0979DA37F1C9C9L, + 0x1EC866B79E0CBA6FL, 0xA9A8C2F6BFE942DBL, 0x18A0522C7E709526L, 0x2153CF2BCCBA9BE3L, + 0x13B374F06526DDB8L, 0x1AA9728970954982L, 0x1F8587E7083E2F8CL, 0xF775840F1A88759DL, + 0x19379FEC0698260AL, 0x5F9136727BA05E17L, 0x142C7FF0054684D5L, 0x1940F85B9619E4DFL, + 0x1023998CD1053710L, 0xE100C6AFAB47EA4CL, 0x19D28F47B4D524E7L, 0xCE67A44C453FDD47L, + 0x14A8729FC3DDB71FL, 0xD852E9D69DCCB106L, 0x1086C219697E2C19L, 0x79DBEE454B0A2738L, + 0x1A71368F0F30468FL, 0x295FE3A211A9D859L, 0x15275ED8D8F36BA5L, 0xBAB31C81A7BB137AL, + 0x10EC4BE0AD8F8951L, 0x6228E39AEC95A92FL, 0x1B13AC9AAF4C0EE8L, 0x9D0E38F7E0EF7517L, + 0x15A956E225D67253L, 0xB0D82D931A592A79L, 0x11544581B7DEC1DCL, 0x8D79BE0F4847552EL, + 0x1BBA08CF8C979C94L, 0x158F967EDA0BBB7CL, 0x162E6D72D6DFB076L, 0x77A611FF14D62F97L, + 0x11BEBDF578B2F391L, 0xF951A7FF43DE8C79L, 0x1C6463225AB7EC1CL, 0xC21C3FFED2FDAD8EL, + 0x16B6B5B5155FF017L, 0x01B0333242648AD8L, 0x122BC490DDE659ACL, 0x0159C28E9B83A246L, + 0x1D12D41AFCA3C2ACL, 0xCEF604175F3903A3L, 0x17424348CA1C9BBDL, 0x725E69AC4C2D9C83L, + 0x129B69070816E2FDL, 0xF5185489D68AE39CL, 0x1DC574D80CF16B2FL, 0xEE8D540FBDAB05C6L, + 0x17D12A4670C1228CL, 0xBED77672FE226B05L, 0x130DBB6B8D674ED6L, 0xFF12C528CB4EBC04L, + 0x1E7C5F127BD87E24L, 0xCB513B74787DF9A0L, 0x18637F41FCAD31B7L, 0x090DC929F9FE614DL, + 0x1382CC34CA2427C5L, 0xA0D7D42194CB810AL, 0x1F37AD21436D0C6FL, 0x67BFB9CF5478CE77L, + 0x18F9574DCF8A7059L, 0x1FCC94A5DD2D71F9L, 0x13FAAC3E3FA1F37AL, 0x7FD6DD517DBDF4C7L, + 0x1FF779FD329CB8C3L, 0xFFBE2EE8C92FEE0BL, 0x1992C7FDC216FA36L, 0x6631BF20A0F324D6L, + 0x14756CCB01ABFB5EL, 0xB827CC1A1A5C1D78L, 0x105DF0A267BCC918L, 0x935309AE7B7CE460L, + 0x1A2FE76A3F9474F4L, 0x1EEB42B0C594A099L, 0x14F31F8832DD2A5CL, 0xE58902270476E6E1L, + 0x10C27FA028B0EEB0L, 0xB7A0CE859D2BEBE7L, 0x1AD0CC33744E4AB4L, 0x59014A6F61DFDFD8L, + 0x1573D68F903EA229L, 0xE0CDD525E7E64CADL, 0x11297872D9CBB4EEL, 0x4D7177518651D6F1L, + 0x1B758D848FAC54B0L, 0x7BE8BEE8D6E957E8L, 0x15F7A46A0C89DD59L, 0xFCBA3253DF211320L, + 0x1192E9EE706E4AAEL, 0x63C8284318E74280L, 0x1C1E43171A4A1117L, 0x060D0D3827D86A66L, + 0x167E9C127B6E7412L, 0x6B3DA42CECAD21EBL, 0x11FEE341FC585CDBL, 0x88FE1CF0BD574E56L, + 0x1CCB0536608D615FL, 0x419694B462254A23L, 0x1708D0F84D3DE77FL, 0x67ABAA29E81DD4E9L, + 0x126D73F9D764B932L, 0xB95621BB2017DD87L, 0x1D7BECC2F23AC1EAL, 0xC223692B668C95A5L, + 0x179657025B6234BBL, 0xCE82BA891ED6DE1DL, 0x12DEAC01E2B4F6FCL, 0xA53562074BDF1818L, + 0x1E3113363787F194L, 0x3B889CD87964F359L, 0x18274291C6065ADCL, 0xFC6D4A46C783F5E1L, + 0x13529BA7D19EAF17L, 0x30576E9F06032B1AL, 0x1EEA92A61C311825L, 0x1A257DCB3CD1DE90L, + 0x18BBA884E35A79B7L, 0x481DFE3C30A7E540L, 0x13C9539D82AEC7C5L, 0xD34B31C9C0865100L, + 0x1FA885C8D117A609L, 0x5211E942CDA3B4CDL, 0x19539E3A40DFB807L, 0x74DB21023E1C90A4L, + 0x1442E4FB67196005L, 0xF715B401CB4A0D50L, 0x103583FC527AB337L, 0xF8DE299B09080AA7L, + 0x19EF3993B72AB859L, 0x8E304291A80CDDD7L, 0x14BF6142F8EEF9E1L, 0x3E8D020E200A4B13L, + 0x10991A9BFA58C7E7L, 0x653D9B3E80083C0FL, 0x1A8E90F9908E0CA5L, 0x6EC8F864000D2CE4L, + 0x153EDA614071A3B7L, 0x8BD3F9E999A423EAL, 0x10FF151A99F482F9L, 0x3CA994BAE1501CBBL, + 0x1B31BB5DC320D18EL, 0xC775BAC49BB3612BL, 0x15C162B168E70E0BL, 0xD2C4956A16291A89L, + 0x11678227871F3E6FL, 0xDBD0778811BA7BA1L, 0x1BD8D03F3E9863E6L, 0x2C80BF401C5D929BL, + 0x16470CFF6546B651L, 0xBD33CC3349E47549L, 0x11D270CC51055EA7L, 0xCA8FD68F6E505DD4L, + 0x1C83E7AD4E6EFDD9L, 0x4419574BE3B3C953L, 0x16CFEC8AA52597E1L, 0x0347790982F63AA9L, + 0x123FF06EEA847980L, 0xCF6C60D468C4FBBAL, 0x1D331A4B10D3F59AL, 0xE57A34870E07F92AL, + 0x175C1508DA432AE2L, 0x512E906C0B399422L, 0x12B010D3E1CF5581L, 0xDA8BA6BCD5C7A9B5L, + 0x1DE6815302E5559CL, 0x90DF712E22D90F87L, 0x17EB9AA8CF1DDE16L, 0xDA4C5A8B4F140C6CL, + 0x1322E220A5B17E78L, 0xAEA37BA2A5A9A38AL, 0x1E9E369AA2B59727L, 0x7DD25F6AA2A905A9L, + 0x187E92154EF7AC1FL, 0x97DB7F888220D154L, 0x139874DDD8C6234CL, 0x797C6606CE80A777L, + 0x1F5A549627A36BADL, 0x8F2D700AE4010BF1L, 0x191510781FB5EFBEL, 0x0C2459A25000D65AL, + 0x1410D9F9B2F7F2FEL, 0x701D1481D99A4515L, 0x100D7B2E28C65BFEL, 0xC017439B147B6A77L, + 0x19AF2B7D0E0A2CCAL, 0xCCF205C4ED9243F2L, 0x148C22CA71A1BD6FL, 0x0A5B37D0BE0E9CC2L, + 0x10701BD527B4978CL, 0x0848F973CB3EE3CEL, 0x1A4CF9550C5425ACL, 0xDA0E5BEC78649FB0L, + 0x150A6110D6A9B7BDL, 0x7B3EAFF060507FC0L, 0x10D51A73DEEE2C97L, 0x95CBBFF380406633L, + 0x1AEE90B964B04758L, 0xEFAC665266CD7052L, 0x158BA6FAB6F36C47L, 0x2623850EB8A459DBL, + 0x113C85955F29236CL, 0x1E82D0D893B6AE49L, 0x1B9408EEFEA838ACL, 0xFD9E1AF41F8AB075L, + 0x16100725988693BDL, 0x97B1AF29B2D559F7L, 0x11A66C1E139EDC97L, 0xAC8E25BAF5777B2CL, + 0x1C3D79C9B8FE2DBFL, 0x7A7D092B2258C513L, 0x169794A160CB57CCL, 0x61FDA0EF4EAD6A76L, + 0x1212DD4DE7091309L, 0xE7FE1A590BBDEEC5L, 0x1CEAFBAFD80E84DCL, 0xA6635D5B45FCB13AL, + 0x172262F3133ED0B0L, 0x851C4AAF6B308DC8L, 0x1281E8C275CBDA26L, 0xD0E36EF2BC26D7D4L, + 0x1D9CA79D894629D7L, 0xB49F17EAC6A48C86L, 0x17B08617A104EE46L, 0x2A18DFEF0550706BL, + 0x12F39E794D9D8B6BL, 0x54E0B3259DD9F389L, 0x1E5297287C2F4578L, 0x87CDEB6F62F65274L, + 0x18421286C9BF6AC6L, 0xD30B22BF825EA85DL, 0x13680ED23AFF889FL, 0x0F3C1BCC684BB9E4L, + 0x1F0CE4839198DA98L, 0x18602C7A4079296DL, 0x18D71D360E13E213L, 0x46B356C833942124L, + 0x13DF4A91A4DCB4DCL, 0x388F78A029434DB6L, 0x1FCBAA82A1612160L, 0x5A7F2766A86BAF8AL, + 0x196FBB9BB44DB44DL, 0x153285EBB9EFBFA2L, 0x145962E2F6A4903DL, 0xAA8ED189618C994EL, + 0x1047824F2BB6D9CAL, 0xEED8A7A11AD6E10CL, 0x1A0C03B1DF8AF611L, 0x7E27729B5E249B45L, + 0x14D6695B193BF80DL, 0xFE85F549181D4904L, 0x10AB877C142FF9A4L, 0xCB9E5DD4134AA0D0L, + 0x1AAC0BF9B9E65C3AL, 0xDF63C9535211014DL, 0x15566FFAFB1EB02FL, 0x191CA10F74DA6771L, + 0x1111F32F2F4BC025L, 0xADB080D92A4852C1L, 0x1B4FEB7EB212CD09L, 0x15E7348EAA0D5134L, + 0x15D98932280F0A6DL, 0xAB1F5D3EEE710DC4L, 0x117AD428200C0857L, 0xBC1917658B8DA49DL, + 0x1BF7B9D9CCE00D59L, 0x2CF4F23C127C3A94L, 0x165FC7E170B33DE0L, 0xF0C3F4FCDB969543L, + 0x11E6398126F5CB1AL, 0x5A365D9716121103L, 0x1CA38F350B22DE90L, 0x9056FC24F01CE804L, + 0x16E93F5DA2824BA6L, 0xD9DF301D8CE3ECD0L, 0x125432B14ECEA2EBL, 0xE17F59B13D8323DAL, + 0x1D53844EE47DD179L, 0x68CBC2B52F38395CL, 0x177603725064A794L, 0x53D6355DBF602DE3L, + 0x12C4CF8EA6B6EC76L, 0xA9782AB165E68B1CL, 0x1E07B27DD78B13F1L, 0x0F26AAB56FD744FAL, + 0x18062864AC6F4327L, 0x3F52222ABFDF6A62L, 0x1338205089F29C1FL, 0x65DB4E88997F884EL, + 0x1EC033B40FEA9365L, 0x6FC54A7428CC0D4AL, 0x1899C2F673220F84L, 0x596AA1F68709A43BL, + 0x13AE3591F5B4D936L, 0xADEEE7F86C07B696L, 0x1F7D228322BAF524L, 0x497E3FF3E00C5756L, + 0x1930E868E89590E9L, 0xD464FFF64CD6AC45L, 0x14272053ED4473EEL, 0x4383FFF83D7889D1L, + 0x101F4D0FF1038FF1L, 0xCF9CCCC69793A174L, 0x19CBAE7FE805B31CL, 0x7F6147A425B90252L, + 0x14A2F1FFECD15C16L, 0xCC4DD2E9B7C7350FL, 0x10825B3323DAB012L, 0x3D0B0F215FD290D9L, + 0x1A6A2B85062AB350L, 0x61AB4B689950E7C1L, 0x1521BC6A6B555C40L, 0x4E22A2BA1440B967L, + 0x10E7C9EEBC4449CDL, 0x0B4EE894DD009453L, 0x1B0C764AC6D3A948L, 0x1217DA87C800ED51L, + 0x15A391D56BDC876CL, 0xDB46486CA000BDDAL, 0x114FA7DDEFE39F8AL, 0x490506BD4CCD64AFL, + 0x1BB2A62FE638FF43L, 0xA8080AC87AE23AB1L, 0x162884F31E93FF69L, 0x5339A239FBE82EF4L, + 0x11BA03F5B20FFF87L, 0x75C7B4FB2FECF25DL, 0x1C5CD322B67FFF3FL, 0x22D92191E647EA2EL, + 0x16B0A8E891FFFF65L, 0xB57A8141850654F2L, 0x1226ED86DB3332B7L, 0xC4620101373843F5L, + 0x1D0B15A491EB8459L, 0x3A366801F1F39FEEL, 0x173C115074BC69E0L, 0xFB5EB99B27F6198BL, + 0x129674405D6387E7L, 0x2F7EFAE2865E7AD6L, 0x1DBD86CD6238D971L, 0xE597F7D0D6FD9156L, + 0x17CAD23DE82D7AC1L, 0x8479930D78CADAABL, 0x1308A831868AC89AL, 0xD06142712D6F1556L, + 0x1E74404F3DAADA91L, 0x4D686A4EAF182222L, 0x185D003F6488AEDAL, 0xA453883EF279B4E8L, + 0x137D99CC506D58AEL, 0xE9DC6CFF28615D87L, 0x1F2F5C7A1A488DE4L, 0xA960AE650D6895A4L, + 0x18F2B061AEA07183L, 0xBAB3BEB73DED4483L, 0x13F559E7BEE6C136L, 0x2EF6322C318A9D36L + }; + } + + private static long[] initPow5Split() { + return new long[]{ + 0x1000000000000000L, 0x0000000000000000L, 0x1400000000000000L, 0x0000000000000000L, + 0x1900000000000000L, 0x0000000000000000L, 0x1F40000000000000L, 0x0000000000000000L, + 0x1388000000000000L, 0x0000000000000000L, 0x186A000000000000L, 0x0000000000000000L, + 0x1E84800000000000L, 0x0000000000000000L, 0x1312D00000000000L, 0x0000000000000000L, + 0x17D7840000000000L, 0x0000000000000000L, 0x1DCD650000000000L, 0x0000000000000000L, + 0x12A05F2000000000L, 0x0000000000000000L, 0x174876E800000000L, 0x0000000000000000L, + 0x1D1A94A200000000L, 0x0000000000000000L, 0x12309CE540000000L, 0x0000000000000000L, + 0x16BCC41E90000000L, 0x0000000000000000L, 0x1C6BF52634000000L, 0x0000000000000000L, + 0x11C37937E0800000L, 0x0000000000000000L, 0x16345785D8A00000L, 0x0000000000000000L, + 0x1BC16D674EC80000L, 0x0000000000000000L, 0x1158E460913D0000L, 0x0000000000000000L, + 0x15AF1D78B58C4000L, 0x0000000000000000L, 0x1B1AE4D6E2EF5000L, 0x0000000000000000L, + 0x10F0CF064DD59200L, 0x0000000000000000L, 0x152D02C7E14AF680L, 0x0000000000000000L, + 0x1A784379D99DB420L, 0x0000000000000000L, 0x108B2A2C28029094L, 0x0000000000000000L, + 0x14ADF4B7320334B9L, 0x0000000000000000L, 0x19D971E4FE8401E7L, 0x4000000000000000L, + 0x1027E72F1F128130L, 0x8800000000000000L, 0x1431E0FAE6D7217CL, 0xAA00000000000000L, + 0x193E5939A08CE9DBL, 0xD480000000000000L, 0x1F8DEF8808B02452L, 0xC9A0000000000000L, + 0x13B8B5B5056E16B3L, 0xBE04000000000000L, 0x18A6E32246C99C60L, 0xAD85000000000000L, + 0x1ED09BEAD87C0378L, 0xD8E6400000000000L, 0x13426172C74D822BL, 0x878FE80000000000L, + 0x1812F9CF7920E2B6L, 0x6973E20000000000L, 0x1E17B84357691B64L, 0x03D0DA8000000000L, + 0x12CED32A16A1B11EL, 0x8262889000000000L, 0x178287F49C4A1D66L, 0x22FB2AB400000000L, + 0x1D6329F1C35CA4BFL, 0xABB9F56100000000L, 0x125DFA371A19E6F7L, 0xCB54395CA0000000L, + 0x16F578C4E0A060B5L, 0xBE2947B3C8000000L, 0x1CB2D6F618C878E3L, 0x2DB399A0BA000000L, + 0x11EFC659CF7D4B8DL, 0xFC90400474400000L, 0x166BB7F0435C9E71L, 0x7BB4500591500000L, + 0x1C06A5EC5433C60DL, 0xDAA16406F5A40000L, 0x118427B3B4A05BC8L, 0xA8A4DE8459868000L, + 0x15E531A0A1C872BAL, 0xD2CE16256FE82000L, 0x1B5E7E08CA3A8F69L, 0x87819BAECBE22800L, + 0x111B0EC57E6499A1L, 0xF4B1014D3F6D5900L, 0x1561D276DDFDC00AL, 0x71DD41A08F48AF40L, + 0x1ABA4714957D300DL, 0x0E549208B31ADB10L, 0x10B46C6CDD6E3E08L, 0x28F4DB456FF0C8EAL, + 0x14E1878814C9CD8AL, 0x33321216CBECFB25L, 0x1A19E96A19FC40ECL, 0xBFFE969C7EE839EEL, + 0x105031E2503DA893L, 0xF7FF1E21CF512435L, 0x14643E5AE44D12B8L, 0xF5FEE5AA43256D42L, + 0x197D4DF19D605767L, 0x337E9F14D3EEC893L, 0x1FDCA16E04B86D41L, 0x005E46DA08EA7AB7L, + 0x13E9E4E4C2F34448L, 0xA03AEC4845928CB3L, 0x18E45E1DF3B0155AL, 0xC849A75A56F72FDFL, + 0x1F1D75A5709C1AB1L, 0x7A5C1130ECB4FBD7L, 0x13726987666190AEL, 0xEC798ABE93F11D66L, + 0x184F03E93FF9F4DAL, 0xA797ED6E38ED64C0L, 0x1E62C4E38FF87211L, 0x517DE8C9C728BDF0L, + 0x12FDBB0E39FB474AL, 0xD2EEB17E1C7976B6L, 0x17BD29D1C87A191DL, 0x87AA5DDDA397D463L, + 0x1DAC74463A989F64L, 0xE994F5550C7DC97CL, 0x128BC8ABE49F639FL, 0x11FD195527CE9DEEL, + 0x172EBAD6DDC73C86L, 0xD67C5FAA71C24569L, 0x1CFA698C95390BA8L, 0x8C1B77950E32D6C3L, + 0x121C81F7DD43A749L, 0x57912ABD28DFC63AL, 0x16A3A275D494911BL, 0xAD75756C7317B7C9L, + 0x1C4C8B1349B9B562L, 0x98D2D2C78FDDA5BBL, 0x11AFD6EC0E14115DL, 0x9F83C3BCB9EA8795L, + 0x161BCCA7119915B5L, 0x0764B4ABE865297AL, 0x1BA2BFD0D5FF5B22L, 0x493DE1D6E27E73D8L, + 0x1145B7E285BF98F5L, 0x6DC6AD264D8F0867L, 0x159725DB272F7F32L, 0xC938586FE0F2CA81L, + 0x1AFCEF51F0FB5EFFL, 0x7B866E8BD92F7D21L, 0x10DE1593369D1B5FL, 0xAD34051767BDAE35L, + 0x15159AF804446237L, 0x9881065D41AD19C2L, 0x1A5B01B605557AC5L, 0x7EA147F492186033L, + 0x1078E111C3556CBBL, 0x6F24CCF8DB4F3C20L, 0x14971956342AC7EAL, 0x4AEE003712230B28L, + 0x19BCDFABC13579E4L, 0xDDA98044D6ABCDF1L, 0x10160BCB58C16C2FL, 0x0A89F02B062B60B7L, + 0x141B8EBE2EF1C73AL, 0xCD2C6C35C7B638E5L, 0x1922726DBAAE3909L, 0x8077874339A3C71EL, + 0x1F6B0F092959C74BL, 0xE0956914080CB8E5L, 0x13A2E965B9D81C8FL, 0x6C5D61AC8507F38FL, + 0x188BA3BF284E23B3L, 0x4774BA17A649F073L, 0x1EAE8CAEF261ACA0L, 0x1951E89D8FDC6C90L, + 0x132D17ED577D0BE4L, 0x0FD3316279E9C3DAL, 0x17F85DE8AD5C4EDDL, 0x13C7FDBB186434D0L, + 0x1DF67562D8B36294L, 0x58B9FD29DE7D4204L, 0x12BA095DC7701D9CL, 0xB7743E3A2B0E4943L, + 0x17688BB5394C2503L, 0xE5514DC8B5D1DB93L, 0x1D42AEA2879F2E44L, 0xDEA5A13AE3465278L, + 0x1249AD2594C37CEBL, 0x0B2784C4CE0BF38BL, 0x16DC186EF9F45C25L, 0xCDF165F6018EF06EL, + 0x1C931E8AB871732FL, 0x416DBF7381F2AC89L, 0x11DBF316B346E7FDL, 0x88E497A83137ABD6L, + 0x1652EFDC6018A1FCL, 0xEB1DBD923D8596CBL, 0x1BE7ABD3781ECA7CL, 0x25E52CF6CCE6FC7EL, + 0x1170CB642B133E8DL, 0x97AF3C1A40105DCFL, 0x15CCFE3D35D80E30L, 0xFD9B0B20D0147543L, + 0x1B403DCC834E11BDL, 0x3D01CDE904199293L, 0x1108269FD210CB16L, 0x462120B1A28FFB9CL, + 0x154A3047C694FDDBL, 0xD7A968DE0B33FA83L, 0x1A9CBC59B83A3D52L, 0xCD93C3158E00F924L, + 0x10A1F5B813246653L, 0xC07C59ED78C09BB7L, 0x14CA732617ED7FE8L, 0xB09B7068D6F0C2A4L, + 0x19FD0FEF9DE8DFE2L, 0xDCC24C830CACF34DL, 0x103E29F5C2B18BEDL, 0xC9F96FD1E7EC1810L, + 0x144DB473335DEEE9L, 0x3C77CBC661E71E14L, 0x1961219000356AA3L, 0x8B95BEB7FA60E599L, + 0x1FB969F40042C54CL, 0x6E7B2E65F8F91EFFL, 0x13D3E2388029BB4FL, 0xC50CFCFFBB9BB360L, + 0x18C8DAC6A0342A23L, 0xB6503C3FAA82A038L, 0x1EFB1178484134ACL, 0xA3E44B4F95234845L, + 0x135CEAEB2D28C0EBL, 0xE66EAF11BD360D2CL, 0x183425A5F872F126L, 0xE00A5AD62C839076L, + 0x1E412F0F768FAD70L, 0x980CF18BB7A47494L, 0x12E8BD69AA19CC66L, 0x5F0816F752C6C8DDL, + 0x17A2ECC414A03F7FL, 0xF6CA1CB527787B14L, 0x1D8BA7F519C84F5FL, 0xF47CA3E2715699D8L, + 0x127748F9301D319BL, 0xF8CDE66D86D62027L, 0x17151B377C247E02L, 0xF7016008E88BA831L, + 0x1CDA62055B2D9D83L, 0xB4C1B80B22AE923DL, 0x12087D4358FC8272L, 0x50F91306F5AD1B66L, + 0x168A9C942F3BA30EL, 0xE53757C8B3186240L, 0x1C2D43B93B0A8BD2L, 0x9E852DBADFDE7AD0L, + 0x119C4A53C4E69763L, 0xA3133C94CBEB0CC2L, 0x16035CE8B6203D3CL, 0x8BD80BB9FEE5CFF2L, + 0x1B843422E3A84C8BL, 0xAECE0EA87E9F43EFL, 0x1132A095CE492FD7L, 0x4D40C9294F238A76L, + 0x157F48BB41DB7BCDL, 0x2090FB73A2EC6D13L, 0x1ADF1AEA12525AC0L, 0x68B53A508BA78857L, + 0x10CB70D24B7378B8L, 0x417144725748B537L, 0x14FE4D06DE5056E6L, 0x51CD958EED1AE284L, + 0x1A3DE04895E46C9FL, 0xE640FAF2A8619B25L, 0x1066AC2D5DAEC3E3L, 0xEFE89CD7A93D00F8L, + 0x14805738B51A74DCL, 0xEBE2C40D938C4135L, 0x19A06D06E2611214L, 0x26DB7510F86F5182L, + 0x100444244D7CAB4CL, 0x9849292A9B4592F2L, 0x1405552D60DBD61FL, 0xBE5B73754216F7AEL, + 0x1906AA78B912CBA7L, 0xADF25052929CB599L, 0x1F485516E7577E91L, 0x996EE4673743E300L, + 0x138D352E5096AF1AL, 0xFFE54EC0828A6DE0L, 0x18708279E4BC5AE1L, 0xBFDEA270A32D0958L, + 0x1E8CA3185DEB719AL, 0x2FD64B0CCBF84BAEL, 0x1317E5EF3AB32700L, 0x5DE5EEE7FF7B2F4DL, + 0x17DDDF6B095FF0C0L, 0x755F6AA1FF59FB20L, 0x1DD55745CBB7ECF0L, 0x92B7454A7F3079E8L, + 0x12A5568B9F52F416L, 0x5BB28B4E8F7E4C31L, 0x174EAC2E8727B11BL, 0xF29F2E22335DDF3DL, + 0x1D22573A28F19D62L, 0xEF46F9AAC035570CL, 0x123576845997025DL, 0xD58C5C0AB8215668L, + 0x16C2D4256FFCC2F5L, 0x4AEF730D6629AC02L, 0x1C73892ECBFBF3B2L, 0x9DAB4FD0BFB41702L, + 0x11C835BD3F7D784FL, 0xA28B11E277D08E61L, 0x163A432C8F5CD663L, 0x8B2DD65B15C4B1FAL, + 0x1BC8D3F7B3340BFCL, 0x6DF94BF1DB35DE78L, 0x115D847AD000877DL, 0xC4BBCF772901AB0BL, + 0x15B4E5998400A95DL, 0x35EAC354F34215CEL, 0x1B221EFFE500D3B4L, 0x8365742A30129B41L, + 0x10F5535FEF208450L, 0xD21F689A5E0BA109L, 0x1532A837EAE8A565L, 0x06A742C0F58E894BL, + 0x1A7F5245E5A2CEBEL, 0x4851137132F22B9EL, 0x108F936BAF85C136L, 0xED32AC26BFD75B43L, + 0x14B378469B673184L, 0xA87F57306FCD3213L, 0x19E056584240FDE5L, 0xD29F2CFC8BC07E98L, + 0x102C35F729689EAFL, 0xA3A37C1DD7584F1FL, 0x14374374F3C2C65BL, 0x8C8C5B254D2E62E7L, + 0x1945145230B377F2L, 0x6FAF71EEA079FBA0L, 0x1F965966BCE055EFL, 0x0B9B4E6A48987A88L, + 0x13BDF7E0360C35B5L, 0x674111026D5F4C95L, 0x18AD75D8438F4322L, 0xC111554308B71FBBL, + 0x1ED8D34E547313EBL, 0x7155AA93CAE4E7A9L, 0x13478410F4C7EC73L, 0x26D58A9C5ECF10CAL, + 0x1819651531F9E78FL, 0xF08AED437682D4FCL, 0x1E1FBE5A7E786173L, 0xECADA89454238A3BL, + 0x12D3D6F88F0B3CE8L, 0x73EC895CB4963665L, 0x1788CCB6B2CE0C22L, 0x90E7ABB3E1BBC3FEL, + 0x1D6AFFE45F818F2BL, 0x352196A0DA2AB4FEL, 0x1262DFEEBBB0F97BL, 0x0134FE24885AB11FL, + 0x16FB97EA6A9D37D9L, 0xC1823DADAA715D66L, 0x1CBA7DE5054485D0L, 0x31E2CD19150DB4C0L, + 0x11F48EAF234AD3A2L, 0x1F2DC02FAD2890F8L, 0x1671B25AEC1D888AL, 0xA6F9303B9872B536L, + 0x1C0E1EF1A724EAADL, 0x50B77C4A7E8F6283L, 0x1188D357087712ACL, 0x5272ADAE8F199D92L, + 0x15EB082CCA94D757L, 0x670F591A32E004F7L, 0x1B65CA37FD3A0D2DL, 0x40D32F60BF980634L, + 0x111F9E62FE44483CL, 0x4883FD9C77BF03E1L, 0x156785FBBDD55A4BL, 0x5AA4FD0395AEC4D9L, + 0x1AC1677AAD4AB0DEL, 0x314E3C447B1A760FL, 0x10B8E0ACAC4EAE8AL, 0xDED0E5AACCF089CAL, + 0x14E718D7D7625A2DL, 0x96851F15802CAC3CL, 0x1A20DF0DCD3AF0B8L, 0xFC2666DAE037D74BL, + 0x10548B68A044D673L, 0x9D980048CC22E68FL, 0x1469AE42C8560C10L, 0x84FE005AFF2BA033L, + 0x198419D37A6B8F14L, 0xA63D8071BEF6883FL, 0x1FE52048590672D9L, 0xCFCCE08E2EB42A4FL, + 0x13EF342D37A407C8L, 0x21E00C58DD309A71L, 0x18EB0138858D09BAL, 0x2A580F6F147CC10EL, + 0x1F25C186A6F04C28L, 0xB4EE134AD99BF151L, 0x137798F428562F99L, 0x7114CC0EC80176D3L, + 0x18557F31326BBB7FL, 0xCD59FF127A01D487L, 0x1E6ADEFD7F06AA5FL, 0xC0B07ED7188249A9L, + 0x1302CB5E6F642A7BL, 0xD86E4F466F516E0AL, 0x17C37E360B3D351AL, 0xCE89E3180B25C98CL, + 0x1DB45DC38E0C8261L, 0x822C5BDE0DEF3BEFL, 0x1290BA9A38C7D17CL, 0xF15BB96AC8B58576L, + 0x1734E940C6F9C5DCL, 0x2DB2A7C57AE2E6D3L, 0x1D022390F8B83753L, 0x391F51B6D99BA087L, + 0x1221563A9B732294L, 0x03B3931248014455L, 0x16A9ABC9424FEB39L, 0x04A077D6DA01956AL, + 0x1C5416BB92E3E607L, 0x45C895CC9081FAC4L, 0x11B48E353BCE6FC4L, 0x8B9D5D9FDA513CBBL, + 0x1621B1C28AC20BB5L, 0xAE84B507D0E58BE9L, 0x1BAA1E332D728EA3L, 0x1A25E249C51EEEE4L, + 0x114A52DFFC679925L, 0xF057AD6E1B33554EL, 0x159CE797FB817F6FL, 0x6C6D98C9A2002AA2L, + 0x1B04217DFA61DF4BL, 0x4788FEFC0A80354AL, 0x10E294EEBC7D2B8FL, 0x0CB59F5D8690214FL, + 0x151B3A2A6B9C7672L, 0xCFE30734E83429A2L, 0x1A6208B50683940FL, 0x83DBC9022241340BL, + 0x107D457124123C89L, 0xB2695DA15568C087L, 0x149C96CD6D16CBACL, 0x1F03B509AAC2F0A8L, + 0x19C3BC80C85C7E97L, 0x26C4A24C1573ACD2L, 0x101A55D07D39CF1EL, 0x783AE56F8D684C04L, + 0x1420EB449C8842E6L, 0x16499ECB70C25F04L, 0x19292615C3AA539FL, 0x9BDC067E4CF2F6C5L, + 0x1F736F9B3494E887L, 0x82D3081DE02FB477L, 0x13A825C100DD1154L, 0xB1C3E512AC1DD0CAL, + 0x18922F31411455A9L, 0xDE34DE57572544FDL, 0x1EB6BAFD91596B14L, 0x55C215ED2CEE963CL, + 0x133234DE7AD7E2ECL, 0xB5994DB43C151DE6L, 0x17FEC216198DDBA7L, 0xE2FFA1214B1A655FL, + 0x1DFE729B9FF15291L, 0xDBBF89699DE0FEB7L, 0x12BF07A143F6D39BL, 0x2957B5E202AC9F32L, + 0x176EC98994F48881L, 0xF3ADA35A8357C6FFL, 0x1D4A7BEBFA31AAA2L, 0x70990C31242DB8BEL, + 0x124E8D737C5F0AA5L, 0x865FA79EB69C9377L, 0x16E230D05B76CD4EL, 0xE7F791866443B855L, + 0x1C9ABD04725480A2L, 0xA1F575E7FD54A66AL, 0x11E0B622C774D065L, 0xA53969B0FE54E802L, + 0x1658E3AB7952047FL, 0x0E87C41D3DEA2203L, 0x1BEF1C9657A6859EL, 0xD229B5248D64AA83L, + 0x117571DDF6C81383L, 0x435A1136D85EEA92L, 0x15D2CE55747A1864L, 0x143095848E76A537L, + 0x1B4781EAD1989E7DL, 0x193CBAE5B2144E84L, 0x110CB132C2FF630EL, 0x2FC5F4CF8F4CB113L, + 0x154FDD7F73BF3BD1L, 0xBBB77203731FDD57L, 0x1AA3D4DF50AF0AC6L, 0x2AA54E844FE7D4ADL, + 0x10A6650B926D66BBL, 0xDAA75112B1F0E4ECL, 0x14CFFE4E7708C06AL, 0xD15125575E6D1E27L, + 0x1A03FDE214CAF085L, 0x85A56EAD360865B1L, 0x10427EAD4CFED653L, 0x7387652C41C53F8FL, + 0x14531E58A03E8BE8L, 0x50693E7752368F72L, 0x1967E5EEC84E2EE2L, 0x64838E1526C4334FL, + 0x1FC1DF6A7A61BA9AL, 0xFDA4719A70754023L, 0x13D92BA28C7D14A0L, 0xDE86C70086494816L, + 0x18CF768B2F9C59C9L, 0x162878C0A7DB9A1BL, 0x1F03542DFB83703BL, 0x5BB296F0D1D280A2L, + 0x1362149CBD322625L, 0x194F9E5683239065L, 0x183A99C3EC7EAFAEL, 0x5FA385EC23EC747FL, + 0x1E494034E79E5B99L, 0xF78C67672CE7919EL, 0x12EDC82110C2F940L, 0x3AB7C0A07C10BB03L, + 0x17A93A2954F3B790L, 0x4965B0C89B14E9C4L, 0x1D9388B3AA30A574L, 0x5BBF1CFAC1DA2434L, + 0x127C35704A5E6768L, 0xB957721CB92856A1L, 0x171B42CC5CF60142L, 0xE7AD4EA3E7726C49L, + 0x1CE2137F74338193L, 0xA198A24CE14F075BL, 0x120D4C2FA8A030FCL, 0x44FF65700CD16499L, + 0x16909F3B92C83D3BL, 0x563F3ECC1005BDBFL, 0x1C34C70A777A4C8AL, 0x2BCF0E7F14072D2FL, + 0x11A0FC668AAC6FD6L, 0x5B61690F6C847C3EL, 0x16093B802D578BCBL, 0xF239C35347A59B4DL, + 0x1B8B8A6038AD6EBEL, 0xEEC83428198F0220L, 0x1137367C236C6537L, 0x553D20990FF96154L, + 0x1585041B2C477E85L, 0x2A8C68BF53F7B9A9L, 0x1AE64521F7595E26L, 0x752F82EF28F5A813L, + 0x10CFEB353A97DAD8L, 0x093DB1D57999890CL, 0x1503E602893DD18EL, 0x0B8D1E4AD7FFEB4FL, + 0x1A44DF832B8D45F1L, 0x8E7065DD8DFFE623L, 0x106B0BB1FB384BB6L, 0xF9063FAA78BFEFD6L, + 0x1485CE9E7A065EA4L, 0xB747CF9516EFEBCBL, 0x19A742461887F64DL, 0xE519C37A5CABE6BEL, + 0x1008896BCF54F9F0L, 0xAF301A2C79EB7037L, 0x140AABC6C32A386CL, 0xDAFC20B798664C44L, + 0x190D56B873F4C688L, 0x11BB28E57E7FDF55L, 0x1F50AC6690F1F82AL, 0x1629F31EDE1FD72BL, + 0x13926BC01A973B1AL, 0x4DDA37F34AD3E67BL, 0x187706B0213D09E0L, 0xE150C5F01D88E01AL, + 0x1E94C85C298C4C59L, 0x19A4F76C24EB1820L, 0x131CFD3999F7AFB7L, 0xB0071AA39712EF14L, + 0x17E43C8800759BA5L, 0x9C08E14C7CD7AAD9L, 0x1DDD4BAA0093028FL, 0x030B199F9C0D958FL, + 0x12AA4F4A405BE199L, 0x61E6F003C1887D7AL, 0x1754E31CD072D9FFL, 0xBA60AC04B1EA9CD8L, + 0x1D2A1BE4048F907FL, 0xA8F8D705DE65440EL, 0x123A516E82D9BA4FL, 0xC99B8663AAFF4A89L, + 0x16C8E5CA239028E3L, 0xBC0267FC95BF1D2BL, 0x1C7B1F3CAC74331CL, 0xAB0301FBBB2EE475L, + 0x11CCF385EBC89FF1L, 0xEAE1E13D54FD4ECAL, 0x1640306766BAC7EEL, 0x659A598CAA3CA27CL, + 0x1BD03C81406979E9L, 0xFF00EFEFD4CBCB1BL, 0x116225D0C841EC32L, 0x3F6095F5E4FF5EF1L, + 0x15BAAF44FA52673EL, 0xCF38BB735E3F36ADL, 0x1B295B1638E7010EL, 0x8306EA5035CF0458L, + 0x10F9D8EDE39060A9L, 0x11E4527221A162B7L, 0x15384F295C7478D3L, 0x565D670EAA09BB65L, + 0x1A8662F3B3919708L, 0x2BF4C0D2548C2A3EL, 0x1093FDD8503AFE65L, 0x1B78F88374D79A67L, + 0x14B8FD4E6449BDFEL, 0x625736A4520D8101L, 0x19E73CA1FD5C2D7DL, 0xFAED044D6690E141L, + 0x103085E53E599C6EL, 0xBCD422B0601A8CC9L, 0x143CA75E8DF0038AL, 0x6C092B5C78212FFBL, + 0x194BD136316C046DL, 0x070B763396297BF9L, 0x1F9EC583BDC70588L, 0x48CE53C07BB3DAF7L, + 0x13C33B72569C6375L, 0x2D80F4584D5068DBL, 0x18B40A4EEC437C52L, 0x78E1316E60A48311L, + }; + } + + /** + * Computes (m * (mul0:mul1)) >> shift, where mul0 is the high 64 bits and + * mul1 is the low 64 bits of a 128-bit multiplier. + *

      + * The 192-bit product is: high0 * 2^128 + (low0 + high1) * 2^64 + low1. + * We extract the upper 128 bits and right-shift by (shift - 64). + */ + // Requires shift in [65, 127] (guaranteed by the Ryu algorithm). + private static long mulShift64(long m, long mul0, long mul1, int shift) { + // Compat.multiplyHigh is signed; correct for unsigned by adding m + // when the other operand is negative (MSB set). + long high1 = Compat.multiplyHigh(m, mul1) + ((mul1 >> 63) & m); + long low0 = m * mul0; + long high0 = Compat.multiplyHigh(m, mul0) + ((mul0 >> 63) & m); + + long mid = low0 + high1; + long carry = Long.compareUnsigned(mid, low0) < 0 ? 1L : 0L; + long high = high0 + carry; + + int s = shift - 64; + return (high << (64 - s)) | (mid >>> s); + } + + // Requires p in [0, 63] because Java's << masks the shift count to 6 bits. + // Callers guarantee p < 63 via the surrounding `q < 63` guard. + private static boolean multipleOfPowerOf2(long value, int p) { + assert p >= 0 && p < 64; + return (value & ((1L << p) - 1)) == 0; + } + + private static boolean multipleOfPowerOf5(long value, int p) { + return pow5Factor(value) >= p; + } + + private static int pow5Factor(long value) { + int count = 0; + while (value > 0) { + if (value % 5 != 0) { + return count; + } + value /= 5; + count++; + } + return count; + } + + private static int pow5bits(int e) { + return ((e * 1217359) >>> 19) + 1; + } + + /** + * Converts a decoded IEEE 754 double to its shortest decimal representation. + * + * @param ieeeMantissa the 52-bit mantissa (without implicit leading 1) + * @param ieeeExponent the biased 11-bit exponent (0-2046; 0 means subnormal) + * @param e10Out single-element array; receives the decimal exponent e10 + * such that value = output * 10^e10 + * @return the decimal significand as a positive long (1-17 digits) + */ + static long d2d(long ieeeMantissa, int ieeeExponent, int[] e10Out) { + int e2; + long m2; + if (ieeeExponent == 0) { + // Subnormal: no implicit leading 1, exponent is 1 (not 0) + e2 = 1 - 1023 - 52; + m2 = ieeeMantissa; + } else { + e2 = ieeeExponent - 1023 - 52; + m2 = ieeeMantissa | (1L << 52); + } + + // Fast path for small integers: if the value is an exact integer in [1, 2^53), + // return it directly. This avoids the 4x-scaling issue in the main Ryu path. + if (e2 >= -52 && e2 <= 0) { + long mask = (1L << -e2) - 1; + if ((m2 & mask) == 0) { + long mantissa = m2 >>> -e2; + int exp = 0; + // Strip trailing decimal zeros + while (mantissa > 0) { + long q = mantissa / 10; + long r = mantissa - 10 * q; + if (r != 0) { + break; + } + mantissa = q; + exp++; + } + e10Out[0] = exp; + return mantissa; + } + } + + // Subtract 2 from e2 to account for the 4x scaling (mv = 4*m2 below). + // This ensures mv * 2^e2 == m2 * 2^(e2_original), so the decimal + // conversion produces the correct (non-4x) significand. + e2 -= 2; + + boolean acceptBounds = (m2 & 1) == 0; + + // Step 2: Determine the interval of valid decimal representations. + long mv = 4 * m2; + long mp = 4 * m2 + 2; + int mmShift = (ieeeMantissa != 0 || ieeeExponent <= 1) ? 1 : 0; + long mm = 4 * m2 - 1 - mmShift; + + // Step 3: Convert to a decimal power base using 128-bit arithmetic. + long vr, vp, vm; + int e10; + boolean vmIsTrailingZeros = false; + boolean vrIsTrailingZeros = false; + + if (e2 >= 0) { + int q = Math.max(0, ((e2 * 78913) >>> 18) - 1); + e10 = q; + int k = DOUBLE_POW5_INV_BITCOUNT + pow5bits(q) - 1; + int i = -e2 + q + k; + int idx = 2 * q; + long mul0 = DOUBLE_POW5_INV_SPLIT[idx]; + long mul1 = DOUBLE_POW5_INV_SPLIT[idx + 1]; + vr = mulShift64(mv, mul0, mul1, i); + vp = mulShift64(mp, mul0, mul1, i); + vm = mulShift64(mm, mul0, mul1, i); + + if (q <= 21) { + if (mv % 5 == 0) { + vrIsTrailingZeros = multipleOfPowerOf5(mv, q); + } else if (acceptBounds) { + vmIsTrailingZeros = multipleOfPowerOf5(mm, q); + } else { + if (multipleOfPowerOf5(mp, q)) { + vp--; + } + } + } + } else { + int q = Math.max(0, ((-e2 * 732923) >>> 20) - 1); + e10 = q + e2; + int i = -e2 - q; + int k = pow5bits(i) - DOUBLE_POW5_BITCOUNT; + int j = q - k; + int idx = 2 * i; + long mul0 = DOUBLE_POW5_SPLIT[idx]; + long mul1 = DOUBLE_POW5_SPLIT[idx + 1]; + vr = mulShift64(mv, mul0, mul1, j); + vp = mulShift64(mp, mul0, mul1, j); + vm = mulShift64(mm, mul0, mul1, j); + + if (q <= 1) { + vrIsTrailingZeros = true; + if (acceptBounds) { + vmIsTrailingZeros = mmShift == 1; + } else { + vp--; + } + } else if (q < 63) { + vrIsTrailingZeros = multipleOfPowerOf2(mv, q); + } + } + + // Step 4: Find the shortest decimal representation in the interval. + int removed = 0; + int lastRemovedDigit = 0; + + if (vmIsTrailingZeros || vrIsTrailingZeros) { + // General case with trailing zeros handling + for (; ; ) { + long vpDiv10 = vp / 10; + long vmDiv10 = vm / 10; + if (vpDiv10 <= vmDiv10) { + break; + } + long vrDiv10 = vr / 10; + boolean vmMod10IsZero = vm - 10 * vmDiv10 == 0; + vmIsTrailingZeros &= vmMod10IsZero; + vrIsTrailingZeros &= lastRemovedDigit == 0; + lastRemovedDigit = (int) (vr - 10 * vrDiv10); + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + + if (vmIsTrailingZeros) { + for (; ; ) { + long vmDiv10 = vm / 10; + if (vm - 10 * vmDiv10 != 0) { + break; + } + long vpDiv10 = vp / 10; + long vrDiv10 = vr / 10; + vrIsTrailingZeros &= lastRemovedDigit == 0; + lastRemovedDigit = (int) (vr - 10 * vrDiv10); + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + } + + if (vrIsTrailingZeros && lastRemovedDigit == 5 && (vr & 1) == 0) { + // Round even + lastRemovedDigit = 4; + } + + long output = vr; + if ((vr == vm && (!acceptBounds || !vmIsTrailingZeros)) || lastRemovedDigit >= 5) { + output++; + } + e10Out[0] = e10 + removed; + return output; + } else { + // Optimized loop when no trailing zeros + boolean roundUp = false; + long vpDiv100 = vp / 100; + long vmDiv100 = vm / 100; + if (vpDiv100 > vmDiv100) { + long vrDiv100 = vr / 100; + long vrMod100 = vr - 100 * vrDiv100; + roundUp = vrMod100 >= 50; + vr = vrDiv100; + vp = vpDiv100; + vm = vmDiv100; + removed += 2; + } + + for (; ; ) { + long vpDiv10 = vp / 10; + long vmDiv10 = vm / 10; + if (vpDiv10 <= vmDiv10) { + break; + } + long vrDiv10 = vr / 10; + roundUp = (vr - 10 * vrDiv10) >= 5; + vr = vrDiv10; + vp = vpDiv10; + vm = vmDiv10; + removed++; + } + + long output = vr + (vr == vm || roundUp ? 1 : 0); + e10Out[0] = e10 + removed; + return output; + } + } + + /** + * Returns the number of decimal digits in a value known to fit in 17 digits. + */ + static int decimalLength17(long v) { + if (v >= 10_000_000_000_000_000L) return 17; + if (v >= 1_000_000_000_000_000L) return 16; + if (v >= 100_000_000_000_000L) return 15; + if (v >= 10_000_000_000_000L) return 14; + if (v >= 1_000_000_000_000L) return 13; + if (v >= 100_000_000_000L) return 12; + if (v >= 10_000_000_000L) return 11; + if (v >= 1_000_000_000L) return 10; + if (v >= 100_000_000L) return 9; + if (v >= 10_000_000L) return 8; + if (v >= 1_000_000L) return 7; + if (v >= 100_000L) return 6; + if (v >= 10_000L) return 5; + if (v >= 1_000L) return 4; + if (v >= 100L) return 3; + if (v >= 10L) return 2; + return 1; + } +} diff --git a/core/src/main/java/io/questdb/client/std/Unsafe.java b/core/src/main/java/io/questdb/client/std/Unsafe.java index 1382a1d90..9b115be1a 100644 --- a/core/src/main/java/io/questdb/client/std/Unsafe.java +++ b/core/src/main/java/io/questdb/client/std/Unsafe.java @@ -210,8 +210,8 @@ private static long AccessibleObject_override_fieldOffset() { // object headers (JEP 450, enabled by default in JDK 27) shrink the // header from 12 to 8 bytes, and a hard-coded 12/16 would then point // inside the header -- the Unsafe write to `override` would silently - // miss and setAccessible() would have no effect (surfacing as - // IllegalAccessError from the FdBig double-formatting bridge on JDK 27). + // miss and setAccessible() would have no effect, so makeAccessible() callers + // would fail with IllegalAccessError on JDK 27. // The probe tracks compact (8), compressed (12), uncompressed (16) and // 32-bit (8) layouts automatically. return firstFieldBoundaryOffset(); diff --git a/core/src/main/java11/io/questdb/client/std/Compat.java b/core/src/main/java11/io/questdb/client/std/Compat.java index 86635036e..f6d39aba4 100644 --- a/core/src/main/java11/io/questdb/client/std/Compat.java +++ b/core/src/main/java11/io/questdb/client/std/Compat.java @@ -45,6 +45,15 @@ public static long currentPid() { } } + /** + * Returns the most significant 64 bits of the 128-bit product of two longs. + * Delegates to {@code Math.multiplyHigh()} on Java 9+, which HotSpot + * intrinsifies; the Java 8 variant computes the same result by hand. + */ + public static long multiplyHigh(long x, long y) { + return Math.multiplyHigh(x, y); + } + /** * Spin-loop pause hint. Delegates to {@code Thread.onSpinWait()} on Java 9+; * the Java 8 variant is a no-op (no such hint exists there). diff --git a/core/src/main/java11/io/questdb/client/std/FdBig.java b/core/src/main/java11/io/questdb/client/std/FdBig.java deleted file mode 100644 index e6d69490a..000000000 --- a/core/src/main/java11/io/questdb/client/std/FdBig.java +++ /dev/null @@ -1,207 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.reflect.Method; - -/** - * Thin bridge over the JDK's internal {@code jdk.internal.math.FDBigInteger}, - * used only by the {@link Numbers} double-to-string slow path. This is the - * Java 9+ variant; the parallel copy under {@code src/main/java8} targets - * {@code sun.misc.FDBigInteger} directly. Keeping the type behind this wrapper - * lets {@code Numbers} carry a single, JDK-agnostic copy of the algorithm. - *

      - * The bignum class is never named in source. Up to JDK 25 it was a - * {@code public} class in a non-exported package, so a compile-time - * {@code --add-exports} plus a reflective runtime export was enough to reach - * it. JDK 26 (JDK-8366017) made the class itself package-private, and no - * module export can make a non-public type accessible. Instead, every method - * is resolved once into a {@code static final} {@link MethodHandle}: - *

        - *
      1. {@link Class#forName(String)} loads the class; loading performs no - * accessibility check.
        2. - *
      2. {@link Class#getDeclaredMethod} finds the member; lookup performs no - * accessibility check either.
        3. - *
      3. {@link Unsafe#makeAccessible} flips {@code AccessibleObject.override} - * directly, bypassing {@code setAccessible}'s module check.
        4. - *
      4. {@link MethodHandles.Lookup#unreflect} sees the override flag and - * resolves through the JDK's trusted {@code IMPL_LOOKUP}, which skips access - * checks entirely and yields a plain direct method handle.
        5. - *
      - * The handles are {@code static final} and invoked via {@code invokeExact} - * with erased ({@code Object}) signatures, so the JIT treats them as - * constants and inlines the calls: the slow path costs the same as a direct - * call did before. The technique works on JDK 9 through 27. - *

      - * JDK 27 needs no change here, but it exposed a latent bug in - * {@link Unsafe#makeAccessible}: compact object headers (JEP 450, enabled by - * default in JDK 27) shrink the object header to 8 bytes, so the - * {@code override} field that step 3 writes moved from offset 12/16 to 8. - * {@code Unsafe} now derives that offset by measuring the first-field boundary - * instead of hard-coding it, which tracks compact, compressed, uncompressed - * and 32-bit layouts alike. The remaining long-term liability is - * {@code sun.misc.Unsafe} itself being removed from a future JDK; the durable - * answer then is a self-contained bignum that needs no JDK-internal access at - * all. The {@code mrjar-smoke} 27-ea CI job guards the current behaviour. - */ -final class FdBig { - private static final MethodHandle ADD_AND_CMP; // (Object, Object, Object) int - private static final MethodHandle CMP; // (Object, Object) int - private static final MethodHandle GET_NORMALIZATION_BIAS; // (Object) int - private static final MethodHandle LEFT_SHIFT; // (Object, int) Object - private static final MethodHandle MULT_BY_10; // (Object) Object - private static final MethodHandle QUO_REM_ITERATION; // (Object, Object) int - private static final MethodHandle VALUE_OF_MUL_POW52; // (long, int, int) Object - private static final MethodHandle VALUE_OF_POW52; // (int, int) Object - - private final Object value; - - private FdBig(Object value) { - this.value = value; - } - - static FdBig valueOfMulPow52(long value, int p5, int p2) { - try { - return new FdBig((Object) VALUE_OF_MUL_POW52.invokeExact(value, p5, p2)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - static FdBig valueOfPow52(int p5, int p2) { - try { - return new FdBig((Object) VALUE_OF_POW52.invokeExact(p5, p2)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int addAndCmp(FdBig x, FdBig y) { - try { - return (int) ADD_AND_CMP.invokeExact(value, x.value, y.value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int cmp(FdBig other) { - try { - return (int) CMP.invokeExact(value, other.value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int getNormalizationBias() { - try { - return (int) GET_NORMALIZATION_BIAS.invokeExact(value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - FdBig leftShift(int shift) { - try { - return new FdBig((Object) LEFT_SHIFT.invokeExact(value, shift)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - FdBig multBy10() { - try { - return new FdBig((Object) MULT_BY_10.invokeExact(value)); - } catch (Throwable t) { - throw rethrow(t); - } - } - - int quoRemIteration(FdBig s) { - try { - return (int) QUO_REM_ITERATION.invokeExact(value, s.value); - } catch (Throwable t) { - throw rethrow(t); - } - } - - /** - * Resolves {@code owner.name(params)} to a method handle of type - * {@code erased} (the bignum type replaced by {@code Object}), without the - * caller needing access to {@code owner}. {@code unreflect} yields a direct - * handle; {@link MethodHandle#asType} then conforms it to the erased type. - * For the handles that narrow an {@code Object} parameter back to the bignum - * type that wraps the direct handle in a cast-inserting adapter (a - * {@code BoundMethodHandle}); only the two {@code valueOf*} handles, which - * merely widen the return to {@code Object}, stay direct. Either way C2 - * inlines the chain and folds the casts, so every call costs the same as a - * direct call and allocates nothing (verified by JMH {@code gc.alloc.rate.norm}). - */ - private static MethodHandle handle( - MethodHandles.Lookup lookup, - Class owner, - String name, - MethodType erased, - Class... params - ) throws ReflectiveOperationException { - Method m = owner.getDeclaredMethod(name, params); - // Sets AccessibleObject.override without setAccessible()'s module - // check. Lookup.unreflect() then treats the member as pre-authorised - // and resolves it via IMPL_LOOKUP, i.e. with no access check at all. - Unsafe.makeAccessible(m); - return lookup.unreflect(m).asType(erased); - } - - private static RuntimeException rethrow(Throwable t) { - if (t instanceof RuntimeException) { - return (RuntimeException) t; - } - if (t instanceof Error) { - throw (Error) t; - } - return new IllegalStateException(t); - } - - static { - try { - final Class big = Class.forName("jdk.internal.math.FDBigInteger"); - final MethodHandles.Lookup lookup = MethodHandles.lookup(); - final Class obj = Object.class; - final Class i32 = int.class; - VALUE_OF_MUL_POW52 = handle(lookup, big, "valueOfMulPow52", MethodType.methodType(obj, long.class, i32, i32), long.class, i32, i32); - VALUE_OF_POW52 = handle(lookup, big, "valueOfPow52", MethodType.methodType(obj, i32, i32), i32, i32); - ADD_AND_CMP = handle(lookup, big, "addAndCmp", MethodType.methodType(i32, obj, obj, obj), big, big); - CMP = handle(lookup, big, "cmp", MethodType.methodType(i32, obj, obj), big); - GET_NORMALIZATION_BIAS = handle(lookup, big, "getNormalizationBias", MethodType.methodType(i32, obj)); - LEFT_SHIFT = handle(lookup, big, "leftShift", MethodType.methodType(obj, obj, i32), i32); - MULT_BY_10 = handle(lookup, big, "multBy10", MethodType.methodType(obj, obj)); - QUO_REM_ITERATION = handle(lookup, big, "quoRemIteration", MethodType.methodType(i32, obj, obj), big); - } catch (ReflectiveOperationException e) { - throw new ExceptionInInitializerError(e); - } - } -} diff --git a/core/src/main/java8/io/questdb/client/std/Compat.java b/core/src/main/java8/io/questdb/client/std/Compat.java index 843b4aa8a..1a9459e28 100644 --- a/core/src/main/java8/io/questdb/client/std/Compat.java +++ b/core/src/main/java8/io/questdb/client/std/Compat.java @@ -59,6 +59,25 @@ public static long currentPid() { * Spin-loop pause hint. No-op on Java 8 — {@code Thread.onSpinWait()} does * not exist there. The parallel Java 9+ variant delegates to it. */ + /** + * Returns the most significant 64 bits of the 128-bit product of two longs. + * Java 8 has no {@code Math.multiplyHigh()}, so this splits both operands + * into 32-bit halves and reassembles the high word from the partial + * products. The Java 9+ variant delegates to the intrinsic instead. + */ + public static long multiplyHigh(long x, long y) { + long x1 = x >> 32; + long x2 = x & 0xFFFFFFFFL; + long y1 = y >> 32; + long y2 = y & 0xFFFFFFFFL; + long z2 = x2 * y2; + long t = x1 * y2 + (z2 >>> 32); + long z1 = t & 0xFFFFFFFFL; + long z0 = t >> 32; + z1 += x2 * y1; + return x1 * y1 + z0 + (z1 >> 32); + } + public static void onSpinWait() { } } diff --git a/core/src/main/java8/io/questdb/client/std/FdBig.java b/core/src/main/java8/io/questdb/client/std/FdBig.java deleted file mode 100644 index a3c2e0822..000000000 --- a/core/src/main/java8/io/questdb/client/std/FdBig.java +++ /dev/null @@ -1,75 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.std; - -import sun.misc.FDBigInteger; - -/** - * Thin bridge over the JDK's internal {@code FDBigInteger}, used only by the - * {@link Numbers} double-to-string slow path. This is the Java 8 variant: the - * bignum class lives in {@code sun.misc}. The parallel copy under - * {@code src/main/java11} targets {@code jdk.internal.math.FDBigInteger}. The - * public surface is identical across both JDKs, so {@code Numbers} carries a - * single, JDK-agnostic copy of the algorithm. - */ -final class FdBig { - private final FDBigInteger value; - - private FdBig(FDBigInteger value) { - this.value = value; - } - - static FdBig valueOfMulPow52(long value, int p5, int p2) { - return new FdBig(FDBigInteger.valueOfMulPow52(value, p5, p2)); - } - - static FdBig valueOfPow52(int p5, int p2) { - return new FdBig(FDBigInteger.valueOfPow52(p5, p2)); - } - - int addAndCmp(FdBig x, FdBig y) { - return value.addAndCmp(x.value, y.value); - } - - int cmp(FdBig other) { - return value.cmp(other.value); - } - - int getNormalizationBias() { - return value.getNormalizationBias(); - } - - FdBig leftShift(int shift) { - return new FdBig(value.leftShift(shift)); - } - - FdBig multBy10() { - return new FdBig(value.multBy10()); - } - - int quoRemIteration(FdBig s) { - return value.quoRemIteration(s.value); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java b/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java index f38ebf4d2..e1e209afc 100644 --- a/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java +++ b/core/src/test/java/io/questdb/client/test/cairo/TableUtilsTest.java @@ -82,8 +82,13 @@ public void testIsValidTableName() { Assert.assertFalse(TableUtils.isValidTableName("\\..", 127)); Assert.assertFalse(TableUtils.isValidTableName("/..", 127)); Assert.assertFalse(TableUtils.isValidTableName("../", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uFEFFt", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uD800t", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uDB80t", 127)); + Assert.assertFalse(TableUtils.isValidTableName("t\uDFFFt", 127)); Assert.assertTrue(TableUtils.isValidTableName("table name", 127)); + Assert.assertTrue(TableUtils.isValidTableName("t\uD83D\uDE00t", 127)); Assert.assertTrue(TableUtils.isValidTableName("table name.csv", 127)); Assert.assertTrue(TableUtils.isValidTableName("table-name", 127)); Assert.assertTrue(TableUtils.isValidTableName("table_name", 127)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java new file mode 100644 index 000000000..25a9df4b8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientSchemaNegotiationTest.java @@ -0,0 +1,228 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.http.client; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.HttpClientException; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.network.Socket; +import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; + +public class WebSocketClientSchemaNegotiationTest { + @Test + public void testDefaultDoesNotRequestAndIgnoresUnsolicitedConfirmation() throws Exception { + ScriptedSocket socket = new ScriptedSocket("X-QWP-Schema: enabled\r\n"); + try (TestClient client = new TestClient(socket)) { + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertFalse(socket.request().contains("X-QWP-Request-Schema:")); + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + + @Test + public void testRequestedConfirmationAndAbsentLegacyResponse() throws Exception { + ScriptedSocket socket = new ScriptedSocket("X-QWP-Schema: EnAbLeD\r\n", ""); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertTrue(socket.request().contains("X-QWP-Request-Schema: true\r\n")); + Assert.assertTrue(client.isQwpSchemaEnabled()); + + client.disconnect(); + Assert.assertFalse("disconnect must clear per-connection confirmation", client.isQwpSchemaEnabled()); + socket.nextResponse(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertTrue("the one-way request survives reuse", socket.request().contains("X-QWP-Request-Schema: true\r\n")); + Assert.assertFalse("an absent response must not retain stale state", client.isQwpSchemaEnabled()); + } + } + + @Test + public void testRejectsInvalidEmptyAndDuplicateConfirmations() throws Exception { + assertRejected("X-QWP-Schema: disabled\r\n", "Invalid X-QWP-Schema"); + assertRejected("X-QWP-Schema: \r\n", "Invalid X-QWP-Schema"); + assertRejected("X-QWP-Schema: enabled\r\nX-QWP-Schema: enabled\r\n", "Duplicate X-QWP-Schema"); + } + + @Test + public void testDoesNotAcceptPrefixedOrValueSpoofHeaders() throws Exception { + String[] headers = { + "Not-X-QWP-Schema: enabled\r\n", + "X-Comment: X-QWP-Schema: enabled\r\n" + }; + for (String header : headers) { + ScriptedSocket socket = new ScriptedSocket(header); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + client.upgrade("/write/v4", 1000, null); + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + } + + private static void assertRejected(String headers, String message) throws Exception { + ScriptedSocket socket = new ScriptedSocket(headers); + try (TestClient client = new TestClient(socket)) { + client.requestQwpSchema(); + client.connect("localhost", 9000); + try { + client.upgrade("/write/v4", 1000, null); + Assert.fail("expected invalid schema confirmation"); + } catch (HttpClientException expected) { + Assert.assertTrue(expected.getMessage(), expected.getMessage().contains(message)); + } + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + + private static final class ScriptedSocket implements Socket { + private final String[] extraHeaders; + private int responseIndex; + private byte[] response; + private int responsePosition; + private String request; + + private ScriptedSocket(String... extraHeaders) { + this.extraHeaders = extraHeaders; + } + + @Override + public void close() { + } + + @Override + public int getFd() { + return 1; + } + + @Override + public boolean isClosed() { + return false; + } + + void nextResponse() { + responseIndex++; + response = null; + responsePosition = 0; + } + + @Override + public void of(int fd) { + } + + String request() { + return request; + } + + @Override + public int recv(long bufferPtr, int bufferLen) { + if (response == null || responsePosition == response.length) { + return 0; + } + int count = Math.min(bufferLen, response.length - responsePosition); + for (int i = 0; i < count; i++) { + Unsafe.getUnsafe().putByte(bufferPtr + i, response[responsePosition++]); + } + return count; + } + + @Override + public int send(long bufferPtr, int bufferLen) { + byte[] bytes = new byte[bufferLen]; + for (int i = 0; i < bufferLen; i++) bytes[i] = Unsafe.getUnsafe().getByte(bufferPtr + i); + request = new String(bytes, StandardCharsets.US_ASCII); + String key = header(request, "Sec-WebSocket-Key:"); + response = response(key, extraHeaders[responseIndex]); + responsePosition = 0; + return bufferLen; + } + + @Override + public void startTlsSession(CharSequence peerName, SocketReadinessWaiter waiter) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean supportsTls() { + return false; + } + + @Override + public int tlsIO(int readinessFlags) { + return 0; + } + + @Override + public boolean wantsTlsWrite() { + return false; + } + + private static String header(String request, String name) { + int start = request.indexOf(name) + name.length(); + int end = request.indexOf("\r\n", start); + return request.substring(start, end).trim(); + } + + private static byte[] response(String key, String extraHeaders) { + try { + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + String accept = Base64.getEncoder().encodeToString(sha1.digest( + (key + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11").getBytes(StandardCharsets.US_ASCII))); + return ("HTTP/1.1 101 Switching Protocols\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + "Sec-WebSocket-Accept: " + accept + "\r\n" + + extraHeaders + + "\r\n").getBytes(StandardCharsets.US_ASCII); + } catch (Exception e) { + throw new AssertionError(e); + } + } + } + + private static final class TestClient extends WebSocketClient { + private TestClient(ScriptedSocket socket) { + super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index 5d2b99763..9d4d3f137 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -252,6 +252,55 @@ public void testCatchUpCapGapMinEscalationWindowConfigStringNotSupportedForTcp() + "catch_up_cap_gap_min_escalation_window_millis=300000;")); } + @Test + public void testSchemaModeBuilderNotSupportedForTcp() { + assertThrows("schema_mode is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .schemaMode(Sender.SchemaMode.STRICT)); + } + + @Test + public void testSchemaModeConfigStringInvalidRejected() { + assertThrows("invalid schema_mode [value=maybe, allowed-values=[auto, strict, off]]", + () -> Sender.builder("ws::addr=localhost:9000;schema_mode=maybe;")); + } + + @Test + public void testSchemaModeConfigStringNotSupportedForTcp() { + assertThrows("schema_mode is only supported for WebSocket transport", + () -> Sender.builder("tcp::addr=localhost:9009;schema_mode=auto;")); + } + + @Test + public void testSchemaModeConfigStringValuesAreCaseInsensitive() { + Assert.assertEquals("AUTO", + Sender.builder("ws::addr=localhost:9000;schema_mode=Auto;").wsConfigSnapshotForTest().get("schema_mode")); + Assert.assertEquals("OFF", + Sender.builder("ws::addr=localhost:9000;schema_mode=OFF;").wsConfigSnapshotForTest().get("schema_mode")); + Assert.assertNull(Sender.builder("ws::addr=localhost:9000;").wsConfigSnapshotForTest().get("schema_mode")); + } + + @Test + public void testSchemaWaitMillisBuilderNotSupportedForTcp() { + assertThrows("schema_wait_millis is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .schemaWaitMillis(1000)); + } + + @Test + public void testSchemaWaitMillisConfigStringZeroRejected() { + assertThrows("schema_wait_millis must be > 0: 0", + () -> Sender.builder("ws::addr=localhost:9000;schema_wait_millis=0;")); + } + + @Test + public void testSchemaWaitMillisUnsetInSnapshot() { + Assert.assertEquals(-1L, + Sender.builder("ws::addr=localhost:9000;").wsConfigSnapshotForTest().get("schema_wait_millis")); + } + @Test public void testCatchUpCapGapMinEscalationWindowUnsetInSnapshot() { // wsConfigSnapshotForTest puts the raw field, not the resolved default (see diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java new file mode 100644 index 000000000..1217a2d13 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaFeedbackResponseTest.java @@ -0,0 +1,261 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +public class QwpSchemaFeedbackResponseTest { + + @Test + public void testUpdatesAreAtomicAndEveryPrefixIsRejected() { + byte[] valid = updates("orders", "trades"); + WebSocketResponse response = new WebSocketResponse(); + withNative(valid, (ptr, length) -> { + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertEquals(2, response.getSchemaUpdateCount()); + Assert.assertEquals("orders", response.getSchemaUpdateTableName(0)); + Assert.assertEquals(0, response.getSchemaUpdate(0).getRequestId()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, response.getSchemaUpdate(0).getResult()); + for (int n = 0; n < length; n++) { + Assert.assertFalse("prefix " + n, response.readFrom(ptr, n, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.isSchemaInvalidation()); + } + }); + } + + @Test + public void testInvalidationAndFailedReadClearVisibleFeedback() { + WebSocketResponse response = new WebSocketResponse(); + byte[] invalidation = base((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)); + withNative(invalidation, (ptr, length) -> { + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertTrue(response.isSchemaInvalidation()); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.readFrom(ptr, length - 1, true)); + Assert.assertFalse(response.isSchemaInvalidation()); + Assert.assertFalse(response.hasSchemaUpdates()); + }); + } + + @Test + public void testNegotiationModesAndDurableAckRules() { + byte[] updates = updates("t"); + withNative(updates, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, false))); + byte[] reserved = base((byte) (WebSocketResponse.STATUS_OK | 0x40)); + withNative(reserved, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + byte[] durable = new byte[]{(byte) (WebSocketResponse.STATUS_DURABLE_ACK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL), 0, 0}; + withNative(durable, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testDuplicateNamesAreCaseInsensitive() { + byte[] duplicate = updates("foo", "FOO"); + withNative(duplicate, (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertFalse(response.readFrom(ptr, length, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + }); + } + + @Test + public void testNestedEnvelopeAndNonZeroFeedbackRequestIdRejected() { + byte[] nested = updatesWithPayload("t", new byte[]{'Q', 'W', 'P', '1', 1, 0, 0, 0, 0, 0, 0, 0}); + withNative(nested, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + byte[] payload = missingPayload(); + payload[1] = 1; + byte[] nonZero = updatesWithPayload("t", payload); + withNative(nonZero, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testInvalidSchemaColumnNamesRejectWholeAckAndNackWithoutPublishingPrefix() { + byte[] valid = missingPayload(); + byte[][] invalidSchemas = { + knownPayload("x", "x"), + knownPayload("x", "X"), + knownPayload("Ä", "ä"), + knownPayload("a/b"), + knownPayload("a\nb") + }; + for (byte[] invalid : invalidSchemas) { + for (byte status : new byte[]{WebSocketResponse.STATUS_OK, WebSocketResponse.STATUS_SCHEMA_MISMATCH}) { + byte[] feedback = updatesWithPayloads(status, new String[]{"first", "second"}, new byte[][]{valid, invalid}); + withNative(feedback, (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertFalse(response.readFrom(ptr, length, true)); + Assert.assertFalse(response.hasSchemaUpdates()); + Assert.assertFalse(response.isSchemaInvalidation()); + }); + } + } + } + + @Test + public void testNackUpdatesPreserveBaseError() { + byte[] payload = missingPayload(); + byte[] name = "bad_table".getBytes(StandardCharsets.UTF_8); + byte[] message = "bad row".getBytes(StandardCharsets.UTF_8); + ByteBuffer b = ByteBuffer.allocate(11 + message.length + 2 + 2 + name.length + 4 + payload.length) + .order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_SCHEMA_MISMATCH | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + b.putLong(19).putShort((short) message.length).put(message).putShort((short) 1); + b.putShort((short) name.length).put(name).putInt(payload.length).put(payload); + withNative(b.array(), (ptr, length) -> { + WebSocketResponse response = new WebSocketResponse(); + Assert.assertTrue(response.readFrom(ptr, length, true)); + Assert.assertEquals(WebSocketResponse.STATUS_SCHEMA_MISMATCH, response.getStatus()); + Assert.assertEquals(19, response.getSequence()); + Assert.assertEquals("bad row", response.getErrorMessage()); + Assert.assertEquals("bad_table", response.getSchemaUpdateTableName(0)); + }); + } + + @Test + public void testHostileCountLengthAndUtf8RejectedWithoutPublication() { + byte[] count = base((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + count = java.util.Arrays.copyOf(count, count.length + 2); + count[count.length - 2] = (byte) 0xff; + count[count.length - 1] = (byte) 0xff; + withNative(count, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + + byte[] invalidUtf8 = updates("t"); + invalidUtf8[15] = (byte) 0xc0; + withNative(invalidUtf8, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + + byte[] oversized = updates("t"); + ByteBuffer.wrap(oversized).order(ByteOrder.LITTLE_ENDIAN).putInt(16, -1); + withNative(oversized, (ptr, length) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, length, true))); + } + + @Test + public void testOneMiBBoundAppliesOnlyToUpdates() { + int count = 3000; + String prefix = new String(new char[123]).replace('\0', '\u4e00'); + int nameLength = 373; + int length = 11 + count * (2 + nameLength + 8); + Assert.assertTrue(length > QwpSchemaProtocol.MAX_FRAME_SIZE); + byte[] base = new byte[length]; + ByteBuffer b = ByteBuffer.wrap(base).order(ByteOrder.LITTLE_ENDIAN); + b.put(WebSocketResponse.STATUS_OK).putLong(1).putShort((short) count); + for (int i = 0; i < count; i++) { + byte[] name = (prefix + String.format(java.util.Locale.ROOT, "%04d", i)).getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(nameLength, name.length); + b.putShort((short) nameLength).put(name).putLong(i); + } + withNative(base, (ptr, n) -> Assert.assertTrue(new WebSocketResponse().readFrom(ptr, n, false))); + base[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL); + withNative(base, (ptr, n) -> Assert.assertTrue(new WebSocketResponse().readFrom(ptr, n, true))); + byte[] payload = missingPayload(); + byte[] update = java.util.Arrays.copyOf(base, base.length + 2 + 2 + 1 + 4 + payload.length); + ByteBuffer trailer = ByteBuffer.wrap(update).order(ByteOrder.LITTLE_ENDIAN); + // Not chained off position(): that returns Buffer, not ByteBuffer, on Java 8. + trailer.position(base.length); + trailer.putShort((short) 1).putShort((short) 1).put((byte) 't').putInt(payload.length).put(payload); + update[0] = (byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES); + withNative(update, (ptr, n) -> Assert.assertFalse(new WebSocketResponse().readFrom(ptr, n, true))); + } + + private static byte[] base(byte status) { + ByteBuffer b = ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN); + b.put(status).putLong(7).putShort((short) 0); + return b.array(); + } + + private static byte[] missingPayload() { + return ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) QwpSchemaProtocol.RESULT_MISSING).array(); + } + + private static byte[] knownPayload(String... columnNames) { + int size = 26; + for (String name : columnNames) { + size += 2 + name.getBytes(StandardCharsets.UTF_8).length + 6; + } + ByteBuffer b = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) -1).putShort((short) columnNames.length); + for (String name : columnNames) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + b.putShort((short) bytes.length).put(bytes).putInt(5).putShort((short) 0); + } + return b.array(); + } + + private static byte[] updates(String... names) { + return updatesWithPayloads(names, missingPayload()); + } + + private static byte[] updatesWithPayload(String name, byte[] payload) { + return updatesWithPayloads(new String[]{name}, payload); + } + + private static byte[] updatesWithPayloads(String[] names, byte[] payload) { + byte[][] payloads = new byte[names.length][]; + java.util.Arrays.fill(payloads, payload); + return updatesWithPayloads(WebSocketResponse.STATUS_OK, names, payloads); + } + + private static byte[] updatesWithPayloads(byte status, String[] names, byte[][] payloads) { + int size = 13; + for (int i = 0; i < names.length; i++) { + size += 2 + names[i].getBytes(StandardCharsets.UTF_8).length + 4 + payloads[i].length; + } + ByteBuffer b = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (status | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)); + b.putLong(7).putShort((short) 0).putShort((short) names.length); + for (int i = 0; i < names.length; i++) { + String name = names[i]; + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + b.putShort((short) bytes.length).put(bytes).putInt(payloads[i].length).put(payloads[i]); + } + return b.array(); + } + + private static void withNative(byte[] bytes, NativeAssertion assertion) { + long ptr = Unsafe.malloc(bytes.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(ptr + i, bytes[i]); + } + assertion.run(ptr, bytes.length); + } finally { + Unsafe.free(ptr, bytes.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private interface NativeAssertion { + void run(long ptr, int length); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java new file mode 100644 index 000000000..37817e9ea --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java @@ -0,0 +1,4578 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.SenderConnectionEvent; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.LongArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaSenderIntegrationTest { + private static final String DECIMAL_TEXT_CORPUS = + "/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv"; + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testDecimalTextOverloadUsesTargetDecimalAndRecoversRows() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 921, 931, ColumnType.getDecimalType(5, 2)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .decimalColumn("value", "1.20m") + .decimalColumn("value", "not-a-decimal") + .atNow(); + + sender.uuidColumn("failed_b", 11, 12); + try { + sender.decimalColumn("value", "not-a-decimal"); + Assert.fail("expected invalid textual decimal"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.decimalColumn("value", "2.5").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimal64Table( + "events", 921, 931, 2, 120, 250); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDecimalTextOverloadFormatsTextAndPreservesNullNoOps() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(941 + targetType, 951, targetType); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", "00123.4500").atNow(); + sender.decimalColumn("value", "1.2300m").atNow(); + sender.decimalColumn("value", "Infinity") + .decimalColumn("value", "not-a-decimal") + .atNow(); + sender.decimalColumn("value", (CharSequence) null) + .decimalColumn("value", "") + .atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).parsedDecimalTextTable( + "events", 941 + targetType, 951); + } + } + }); + } + + @Test + public void testSmallIntegerSettersUseExactNumericTargetWireAndRecoverRows() throws Exception { + assertMemoryLeak(() -> { + int[] targets = { + ColumnType.BYTE, ColumnType.SHORT, ColumnType.INT, + ColumnType.LONG, ColumnType.FLOAT, ColumnType.DOUBLE + }; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(901 + i, 911 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -1).atNow(); + + sender.byteColumn("value", (byte) 99); + try { + switch (i % 3) { + case 0: + sender.byteColumn("failed_b", (byte) 1); + break; + case 1: + sender.shortColumn("failed_b", (short) 1); + break; + default: + sender.intColumn("failed_b", 1); + break; + } + Assert.fail("expected unsupported small-integer-to-UUID conversion"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) 2).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerNumericTable( + "events", 901 + i, 911 + i, numericWireType(target)); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIntegerTemporalGenerationRemainsDateBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(981, 991, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 11).atNow(); + + handler.targetType = ColumnType.TIMESTAMP_NANO; + handler.version = 992; + sender.table("events"); + try { + sender.stringColumn("value", "1970-01-01T00:00:00.000001Z"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").intColumn("value", 12).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)) + .twoDateAndTimestampBlocks("events", 981, 991, 992); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntegerTemporalTargetsUseExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.DATE, ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(961 + i, 971 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -1).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) 2).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 3).atNow(); + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).integerTemporalTable( + "events", 961 + i, 971 + i, temporalWireType(target)); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testSmallIntegerTextGenerationRemainsSymbolBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1021, 1031, ColumnType.SYMBOL); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 7).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 1032; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 2); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").shortColumn("value", (short) 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)) + .twoSymbolAndVarcharBlocks("events", 1021, 1031, 1032); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testSmallIntegerTextTargetsUseExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}; + for (int i = 0; i < targets.length; i++) { + int target = targets[i]; + LongTextSchemaHandler handler = new LongTextSchemaHandler(1041 + i, 1051 + i, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) -128).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", Short.MAX_VALUE).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerTextTable( + "events", 1041 + i, 1051 + i, target); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testSmallIntegerDecimalGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1061, 1071, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 1).atNow(); + + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 1072; + sender.table("events"); + try { + sender.shortColumn("value", (short) 1_000); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").shortColumn("value", (short) 1_000).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoSmallIntegerDecimalBlocks( + "events", 1061, 1071, 1072); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testSmallIntegerDecimalTargetUsesExactWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1081, 1091, ColumnType.getDecimalType(18, 2)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").byteColumn("value", (byte) 42).atNow(); + + sender.shortColumn("value", (short) 99); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT-to-UUID conversion rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.shortColumn("value", (short) -7).atNow(); + sender.intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", Integer.MAX_VALUE).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).smallIntegerDecimalTable( + "events", 1081, 1091); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntToIpv4GenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 1101, 1111, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").intColumn("value", 1).atNow(); + + handler.targetType = ColumnType.IPv4; + handler.version = 1112; + sender.table("events"); + try { + sender.intColumn("value", 1_000); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").intColumn("value", 1_000).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoIntDecimalAndIpv4Blocks( + "events", 1101, 1111, 1112); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testIntToIpv4UsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1121, 1131, ColumnType.IPv4); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").intColumn("value", Integer.MIN_VALUE).atNow(); + sender.intColumn("value", 0).atNow(); + + sender.intColumn("value", 0x0a000001); + try { + sender.intColumn("failed_b", 1); + Assert.fail("expected INT to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.intColumn("value", 1).atNow(); + sender.intColumn("value", -1).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).intToIpv4Table("events", 1121, 1131); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testNativeDecimalTextCorpusUsesExactVarcharWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaSenderIntegrationTest.class.getResourceAsStream(DECIMAL_TEXT_CORPUS); + Assert.assertNotNull(DECIMAL_TEXT_CORPUS, stream); + boolean[][] sourceTargets = new boolean[3][2]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + DecimalTextVector vector = new DecimalTextVector(line.split("\t", -1)); + sourceTargets[vector.sourceIndex()][vector.targetIndex()] = true; + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 721 + count, 731 + count, vector.targetType()); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events"); + vector.append(sender, "value"); + if (vector.isNull()) { + sender.uuidColumn("failed_b", 11, 12); + } + sender.atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + if (vector.isNull()) { + reader.uuidOnlyTable("events", 721 + count, 731 + count, 11, 12); + } else { + reader.decimalTextTable( + "events", 721 + count, 731 + count, vector.expectedText); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(24, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target pair " + source + '/' + target, + sourceTargets[source][target]); + } + } + }); + } + + @Test + public void testNativeDecimalTextUsesPerValueScalesAcrossMixedWidthsAndRecoversRows() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(781, 791, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .decimalColumn("value", Decimal64.NULL_VALUE) + .decimalColumn("value", new Decimal64(12_340, 2)) + .binaryColumn("value", new byte[]{1}) + .atNow(); + + sender.decimalColumn("value", Decimal256.MAX_VALUE); + try { + sender.decimalColumn("failed_b", new Decimal64(1, 0)); + Assert.fail("expected unsupported DECIMAL64-to-UUID conversion"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.decimalColumn("value", new Decimal64(-125, 3)).atNow(); + sender.table("events").atNow(); + sender.decimalColumn("value", new Decimal256( + 0x161bcca7119915b5L, + 0x0764b4abe8652979L, + 0x7775a5f171950fffL, + 0xffffffffffffffffL, + 76)).atNow(); + sender.decimalColumn("value", new Decimal64(0, 0)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimalTextLifecycleTable("events", 781, 791); + } + }); + } + + @Test + public void testNativeDecimalTextMatchesServerScalarFormattingForRawPublicValues() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(801, 811, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + Decimal64 wrappedScale = new Decimal64(123, 1); + wrappedScale.setScale(257); + sender.table("events") + .decimalColumn("value", new Decimal128(Long.MAX_VALUE, -1L)) + .atNow(); + sender.decimalColumn("value", wrappedScale).atNow(); + sender.decimalColumn("value", new Decimal128(0, 420, 2)).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).decimalTextTable( + "events", + 801, + 811, + "170141183460469231731687303715884105727", + "12.3", + "4.20" + ); + } + }); + } + + @Test + public void testNativeDecimalRollbackAndTargetScaleWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(16, 4); + LongTextSchemaHandler handler = new LongTextSchemaHandler(601, 611, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(150, 2)).atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.decimalColumn("value", Decimal128.fromLong(100_000_000_000_000_000L, 0)); + Assert.fail("expected target precision overflow"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.decimalColumn("value", new Decimal64(25, 2)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).decimal64Table("events", 601, 611, 4, 15_000, 2_500); + } + }); + } + + @Test + public void testNarrowDecimalGenerationRemainsPinnedBeforeWideRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 621, 631, ColumnType.getDecimalType(3, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", Decimal128.fromLong(1, 0)).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 0); + handler.version = 632; + sender.table("events"); + try { + sender.decimalColumn("value", Decimal128.fromLong(1_000, 0)); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", Decimal128.fromLong(1_000, 0)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalBlocks("events", 621, 631, 632); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDecimalScaleGenerationRemainsPinnedBeforeScaleRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 641, 651, ColumnType.getDecimalType(18, 0)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(1, 0)).atNow(); + handler.targetType = ColumnType.getDecimalType(18, 4); + handler.version = 652; + sender.table("events"); + try { + sender.decimalColumn("value", new Decimal64(12_345, 4)); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", new Decimal64(12_345, 4)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimal64ScaleBlocks( + "events", 641, 651, 652); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 661, 671, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 1).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 672; + sender.table("events"); + try { + sender.longColumn("value", 100_000_000_000_000L); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 100_000_000_000_000L).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events", 661, 671, 672); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 681, 691, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "1").atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 692; + sender.table("events"); + try { + sender.stringColumn("value", "100000000000000"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "100000000000000").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events", 681, 691, 692); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testFloatingDecimalWidthAndScaleGenerationRemainPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 701, 711, ColumnType.getDecimalType(18, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").floatColumn("value", 1.0f).atNow(); + handler.targetType = ColumnType.getDecimalType(20, 5); + handler.version = 712; + sender.table("events"); + try { + sender.doubleColumn("value", 100_000_000_000_000.0); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleColumn("value", 100_000_000_000_000.0).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimalWidthAndScaleBlocks("events", 701, 711, 712); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testNativeDecimalTextGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 741, 751, ColumnType.getDecimalType(5, 4)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").decimalColumn("value", new Decimal64(12_340, 4)).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 752; + sender.table("events"); + try { + sender.decimalColumn("value", new Decimal64(250_000, 4)); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").decimalColumn("value", new Decimal64(250_000, 4)).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDecimal64AndVarcharBlocks( + "events", 741, 751, 752); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToGeoHashNullWidthsRollbackAndRecovery() throws Exception { + assertMemoryLeak(() -> { + for (int bits : new int[]{1, 8, 60}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 471 + bits, 481, ColumnType.getGeoHashTypeWithBits(bits)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).nullGeoHashTable("events", 471 + bits, 481, bits); + } + } + + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 551, 561, ColumnType.getGeoHashTypeWithBits(8)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "zz").atNow(); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "a"); + Assert.fail("expected invalid GeoHash text"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "04").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).geoHashTable("events", 551, 561, 8, 0xff, 1); + } + }); + } + + @Test + public void testGeoHashPrecisionGenerationRemainsPinnedBeforeRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler( + 571, 581, ColumnType.getGeoHashTypeWithBits(60)); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "0123456789bc").atNow(); + handler.targetType = ColumnType.getGeoHashTypeWithBits(5); + handler.version = 582; + sender.table("events"); + try { + sender.stringColumn("value", "z"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoGeoHashBlocks("events", 571, 581, 582); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToLong256RollbackAndRecoveryWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(431, 441, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + sender.stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "0x0"); + Assert.fail("expected odd-length LONG256 rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.stringColumn("value", "0x0123456789abcdeffedcba9876543210").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).long256Table("events", 431, 441); + } + }); + } + + @Test + public void testLong256GenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(451, 461, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events") + .stringColumn("value", "0x0123456789abcdeffedcba9876543210") + .atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 462; + sender.table("events"); + try { + sender.stringColumn("value", "not-long256"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "not-long256").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoLong256AndVarcharBlocks("events", 451, 461, 462); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToCharRollbackAndRecoveryWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(391, 401, ColumnType.CHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "A").atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.binaryColumn("value", new byte[]{1}); + Assert.fail("expected BINARY to CHAR rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.stringColumn("value", "\ud83d").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).charTable("events", 391, 401, 'A', '?'); + } + }); + } + + @Test + public void testVarcharGenerationRemainsPinnedBeforeCharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(411, 421, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "native-a").atNow(); + handler.targetType = ColumnType.CHAR; + handler.version = 422; + sender.table("events"); + try { + sender.binaryColumn("value", new byte[]{1}); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").charColumn("value", '\uffff').atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoVarcharAndCharBlocks("events", 411, 421, 422); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testTimestampToTextTargetsRollbackAndRecoverWithoutTableReselection() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(351 + target, 361, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.MICROS).atNow(); + sender.uuidColumn("failed_b", 0xa456426614174000L, 0x123e4567e89b12d3L); + try { + sender.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); + Assert.fail("expected timestamp range error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.timestampColumn("value", 1, ChronoUnit.MILLIS).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).timestampTextTable( + "events", 351 + target, 361, "1970-01-01T00:00:00.001Z"); + } + } + }); + } + + @Test + public void testNativeTimestampGenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(371, 381, ColumnType.TIMESTAMP_NANO); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").timestampColumn("value", Long.MIN_VALUE, ChronoUnit.NANOS).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 382; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 0); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").timestampColumn("value", Long.MAX_VALUE, ChronoUnit.NANOS).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoTimestampAndVarcharBlocks( + "events", 371, 381, 382); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testStringToTimestampTargetsUsePinnedWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(301 + target, 311, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "not-a-timestamp"); + Assert.fail("expected timestamp parse error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.stringColumn("value", "1969-12-31T23:59:59.999999Z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).longTimestampTable( + "events", + 301 + target, + 311, + target, + target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000 + ); + } + } + }); + } + + @Test + public void testStringToDateUsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1141, 1151, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", null); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").stringColumn("failed_b", "123e4567-e89b-12d3-a456-426614174000"); + try { + sender.stringColumn("value", "not-a-date"); + Assert.fail("expected DATE parse error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.stringColumn("value", "1969-12-31").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .stringDateTable("events", 1141, 1151, -86_400_000L); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDateGenerationRemainsPinnedBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(1161, 1171, ColumnType.DATE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "1970-01-01").atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 1172; + sender.table("events"); + try { + sender.stringColumn("value", "native-a"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "native-a").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)) + .twoDateAndVarcharBlocks("events", 1161, 1171, 1172); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleArrayUsesExactTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1181, 1191, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.doubleArray("value", new double[]{99.0}); + Assert.fail("expected array rank mismatch"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events", 1181, 1191); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleArrayGenerationRemainsPinnedAcrossRankChange() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1201, 1211, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + handler.targetType = array1d; + handler.version = 1212; + sender.table("events"); + try { + sender.doubleArray("value", new double[]{3.0, 4.0}); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleArray("value", new double[]{3.0, 4.0}).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)) + .twoDoubleArrayRankBlocks("events", 1201, 1211, 1212); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongArrayRejectionRollsBackWithoutRefresh() throws Exception { + assertMemoryLeak(() -> { + for (int overload = 0; overload < 4; overload++) { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1221, 1231, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + switch (overload) { + case 0: + sender.longArray("value", new long[]{99L}); + break; + case 1: + sender.longArray("value", new long[][]{{99L}}); + break; + case 2: + sender.longArray("value", new long[][][]{{{99L}}}); + break; + default: + try (LongArray array = new LongArray(1)) { + sender.longArray("value", array.append(99L)); + } + break; + } + Assert.fail("expected unsupported LONG_ARRAY setter [overload=" + overload + "]"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.doubleArray("value", new double[][]{{3.0}, {4.0}}).atNow(); + sender.flush(); + // Only the two completed DOUBLE[] rows reach the wire; failed_b is rolled back. + new FrameReader(handler.awaitDataFrame()) + .doubleArrayTable("events", 1221, 1231); + Assert.assertEquals("unsupported input must not refresh [overload=" + overload + "]", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongArrayRejectionKeepsPinnedSnapshotAfterSchemaChange() throws Exception { + assertMemoryLeak(() -> { + int array2d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 2); + int array1d = ColumnType.encodeArrayType(ColumnType.DOUBLE, 1); + LongTextSchemaHandler handler = new LongTextSchemaHandler(1241, 1251, array2d); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleArray("value", new double[][]{{1.0, 2.0}}).atNow(); + handler.targetType = array1d; + handler.version = 1252; + for (int attempt = 0; attempt < 2; attempt++) { + sender.table("events").uuidColumn("failed_b", 11, 12); + try { + sender.longArray("value", new long[]{3L, 4L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("schema changes cannot implement LONG_ARRAY", 1, + handler.describeRequests.get()); + } + // A supported input of the new rank is still rejected against the + // pinned snapshot; the batch does not discover the rank change. + try { + sender.doubleArray("value", new double[]{3.0, 4.0}); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleArray("value", new double[]{3.0, 4.0}).atNow(); + sender.flush(); + // The rebound block holds the recovered row alone: rollback dropped + // failed_b with the rejected row. + new FrameReader(handler.awaitDataFrames(2)) + .twoDoubleArrayRankBlocks("events", 1241, 1251, 1252); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test(timeout = 10_000) + public void testLongArrayRejectionDoesNotDependOnDescribeAvailability() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_wait_millis=100;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.respondToDescribe = false; + sender.table("events").stringColumn("partial", "discard"); + try { + sender.longArray("id", new long[]{99L}); + Assert.fail("expected unsupported LONG_ARRAY setter"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testVarcharGenerationRemainsPinnedBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(321, 331, ColumnType.VARCHAR); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "native-a").atNow(); + handler.targetType = ColumnType.TIMESTAMP_MICRO; + handler.version = 332; + sender.table("events"); + try { + sender.stringColumn("failed_b", "not-a-uuid"); + Assert.fail("expected invalid UUID"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("value", "1970-01-01T00:00:00.000001Z").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoVarcharAndTimestampBlocks( + "events", 321, 331, 332 + ); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongToTimestampTargetsUseRawUnitsAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", Long.MIN_VALUE); + sender.binaryColumn("value", new byte[]{1}).atNow(); + sender.table("events").longColumn("value", -1); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()) + .longTimestampTable("events", 251 + target, 261, target, Long.MAX_VALUE); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongGenerationRemainsNativeBeforeTimestampRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(271, 281, ColumnType.LONG); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").longColumn("value", 11).atNow(); + handler.targetType = ColumnType.TIMESTAMP_NANO; + handler.version = 282; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-long"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 12).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoLongAndTimestampBlocks("events", 271, 281, 282); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDoubleGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(231, 241, ColumnType.DOUBLE); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events").doubleColumn("value", -0.0).atNow(); + handler.targetType = ColumnType.VARCHAR; + handler.version = 242; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-double"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").doubleColumn("value", 1e23).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2)).twoDoubleAndVarcharBlocks("events", 231, 241, 242); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testFloatingToTextTargetsUsePinnedWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + for (String input : new String[]{"FLOAT", "DOUBLE"}) { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(211 + target, 221, target); + try (TestWebSocketServer server = schemaServer(handler); Sender sender = sender(server)) { + sender.table("events"); + if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(0x7fc00002)); + else sender.doubleColumn("value", Double.longBitsToDouble(0x7ff8000000000002L)); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events"); + if ("FLOAT".equals(input)) sender.floatColumn("value", 0.1f); + else sender.doubleColumn("value", 1e23); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected floating row rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + if ("FLOAT".equals(input)) sender.floatColumn("value", Float.intBitsToFloat(1)); + else sender.doubleColumn("value", Double.longBitsToDouble(1)); + sender.atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).floatingTextTable( + "events", 211 + target, 221, target, + "FLOAT".equals(input) ? "0.10000000149011612" : "1.0E23", + "FLOAT".equals(input) ? "1.401298464324817E-45" : "5.0E-324" + ); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + } + }); + } + + @Test + public void testUuidGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(191, 201, ColumnType.UUID); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L) + .atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 202; + sender.table("events"); + try { + sender.stringColumn("value", "not-a-uuid"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("value", 0x2122232425262728L, 0x3132333435363738L).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoUuidAndVarcharBlocks("events", 191, 201, 202); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testUuidToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(171 + target, 181, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").uuidColumn("value", 7, 0); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.uuidColumn("value", 0x0102030405060708L, 0x1112131415161718L).atNow(); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.longTextTable( + "events", + 171 + target, + 181, + target, + "11121314-1516-1718-0102-030405060708" + ); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIpv4TargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.IPv4, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(211 + target, 221, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .ipv4Column("value", 0) + .ipv4Column("value", "not-an-ip") + .atNow(); + + sender.table("events").ipv4Column("value", 0x05060708); + try { + sender.ipv4Column("failed_b", 0x090a0b0c); + Assert.fail("expected IPv4 to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.uuidColumn("failed_b", 1, 2); + try { + sender.ipv4Column("value", "not-an-ip"); + Assert.fail("expected invalid IPv4 text"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + + sender.ipv4Column("value", ".....255.1.2.3......").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).ipv4Table( + "events", 211 + target, 221, target, 0xff010203); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testIpv4GenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(231, 241, ColumnType.IPv4); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").ipv4Column("value", 0x01020304).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 242; + sender.table("events"); + try { + sender.ipv4Column("value", "not-an-ip"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").ipv4Column("value", 0x05060708).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoIpv4AndVarcharBlocks("events", 231, 241, 242); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testGeoHashTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + int[] targets = {geo20, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .geoHashColumn("value", "u33d") + .atNow(); + + sender.table("events").geoHashColumn("value", 0x11111, 20); + try { + sender.geoHashColumn("failed_b", 1, 20); + Assert.fail("expected GEOHASH to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.geoHashColumn("value", 0x12345, 20).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).geoHashTargetTable( + "events", 251 + target, 261, target, 0xd0c6c, 0x12345, 20); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testGeoHashGenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + LongTextSchemaHandler handler = new LongTextSchemaHandler(281, 291, geo20); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").geoHashColumn("value", 0xabcde, 20).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 292; + sender.table("events"); + try { + sender.stringColumn("value", "not-geohash"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").geoHashColumn("value", 0x12345, 20).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoNativeGeoHashAndVarcharBlocks( + "events", 281, 291, 292); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLong256TargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.LONG256, ColumnType.STRING, ColumnType.VARCHAR}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(251 + target, 261, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE) + .long256Column("value", 9, 10, 11, 12) + .atNow(); + + sender.table("events").long256Column("value", 5, 6, 7, 8); + try { + sender.long256Column("failed_b", 9, 10, 11, 12); + Assert.fail("expected LONG256 to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + sender.long256Column("value", 1, 2, 3, 4).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).long256TargetTable( + "events", 251 + target, 261, target, 1, 2, 3, 4); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLong256GenerationRemainsNativeBeforeVarcharRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(281, 291, ColumnType.LONG256); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").long256Column("value", 1, 2, 3, 4).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 292; + sender.table("events"); + try { + sender.stringColumn("value", "not-long256"); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").long256Column("value", 5, 6, 7, 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)).twoNativeLong256AndVarcharBlocks( + "events", 281, 291, 292); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testLongToTextTargetsUsePinnedTargetWireAndRollback() throws Exception { + assertMemoryLeak(() -> { + int[] targets = {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}; + for (int target : targets) { + LongTextSchemaHandler handler = new LongTextSchemaHandler(121 + target, 131, target); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("value", Long.MIN_VALUE); + // MIN is an effective null write, so this unsupported duplicate must be ignored. + sender.binaryColumn("value", new byte[]{1}).atNow(); + + sender.table("events").longColumn("value", 7); + try { + sender.longColumn("failed_b", 9); + Assert.fail("expected LONG to UUID rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + + // The failed middle row is auto-cancelled; continue without selecting the table again. + sender.longColumn("value", Long.MAX_VALUE).atNow(); + sender.flush(); + + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.longTextTable("events", 121 + target, 131, target, Long.toString(Long.MAX_VALUE)); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testLongSymbolGenerationRemainsPinnedAcrossTextRebind() throws Exception { + assertMemoryLeak(() -> { + LongTextSchemaHandler handler = new LongTextSchemaHandler(151, 161, ColumnType.SYMBOL); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("value", 7).atNow(); + + handler.targetType = ColumnType.VARCHAR; + handler.version = 162; + sender.table("events"); + try { + sender.uuidColumn("value", 1, 2); + Assert.fail("expected rejection against the pinned snapshot"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").longColumn("value", 8).atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrames(2)) + .twoSymbolAndVarcharBlocks("events", 151, 161, 162); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testAsymmetricUuidRowsArePinnedAndInvalidMiddleRowRollsBack() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 41, 73); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("id", "11121314-1516-1718-0102-030405060708").atNow(); + sender.table("events"); + try { + sender.stringColumn("failed_b", "must-roll-back"); + sender.stringColumn("id", "not-a-uuid"); + Assert.fail("expected invalid UUID"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + sender.uuidColumn("id", 0x2122232425262728L, 0x3132333435363738L).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + FrameReader reader = new FrameReader(frame); + reader.schemaTable("events", 41, 73, 2, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + Assert.assertEquals(0x2122232425262728L, reader.i64()); + Assert.assertEquals(0x3132333435363738L, reader.i64()); + reader.eof(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testDuplicateSetterSkipsInvalidConversionAndRefresh() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 7, 9); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2); + sender.stringColumn("id", "invalid and ignored").atNow(); + sender.flush(); + Assert.assertNotNull(handler.awaitDataFrame()); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testExplicitAndServerAssignedDesignatedTimestampUseSchemaFrames() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 11, 12); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).at(Instant.ofEpochSecond(3, 4)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + } + }); + } + + @Test + public void testMissingTableInfersColumnsAndUsesCachedAbsence() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("missing").longColumn("x", 7).atNow(); + sender.table("missing").longColumn("x", 8).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("missing", -1, -1, 2, "x", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(8, reader.i64()); + reader.eof(); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testKnownAdoptionWaitsForBatchBoundary() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, 31, 41); + handler.isSchemaFeedbackEnabled = true; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 11).atNow(); + handler.result = QwpSchemaProtocol.RESULT_KNOWN; + // The pending batch keeps its inferred snapshot: a conflicting input + // fails against the inferred LONG and the failed row rolls back whole. + try { + sender.stringColumn("failed_b", "partial"); + sender.stringColumn("id", "11121314-1516-1718-0102-030405060708"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals(1, handler.describeRequests.get()); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").stringColumn("id", "21222324-2526-2728-3132-333435363738").atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrames(2)); + reader.twoInferredAdoptionBlocks("events"); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testInferredTypeConflictDoesNotRefreshAndRetainsInferredRows() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").longColumn("id", 1).atNow(); + try { + sender.stringColumn("id", "different inferred type"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("id", 3).atNow(); + sender.flush(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", -1, -1, 2, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + reader.eof(); + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testAutoUnavailableSchemaUsesLegacyFrame() throws Exception { + assertMemoryLeak(() -> { + for (int result : new int[]{QwpSchemaProtocol.RESULT_UNAVAILABLE, QwpSchemaProtocol.RESULT_TOO_LARGE}) { + SchemaHandler handler = new SchemaHandler(result, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + + new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "value", "legacy"); + } + } + }); + } + + @Test + public void testTooLargeSchemaIsDescribedOnce() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_TOO_LARGE, -1, -1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + // AUTO once re-sent a blocking DESCRIBE for every batch of a table + // too wide to describe; the cached negative answer now serves them. + for (int i = 0; i < 5; i++) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + } + for (byte[] frame : handler.awaitDataFrames(5)) { + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + SchemaHandler strictHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_TOO_LARGE, -1, -1); + try (TestWebSocketServer server = schemaServer(strictHandler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + for (int i = 0; i < 2; i++) { + try { + sender.table("events").stringColumn("value", "strict"); + Assert.fail("STRICT must reject a table too wide to describe"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + } + // STRICT has no legacy rows to earn feedback, so it asks again each time. + Assert.assertEquals(2, strictHandler.describeRequests.get()); + } + }); + } + + @Test + public void testAutoTooLargeSchemaPublishesPendingSchemaBatchBeforeLegacyRow() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.result = QwpSchemaProtocol.RESULT_TOO_LARGE; + + // The new table cannot join the pending schema batch: publish it + // before the legacy row starts a new frame. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(2); + FrameReader schema = new FrameReader(frames.get(0)); + schema.schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(1, schema.i64()); + Assert.assertEquals(2, schema.i64()); + schema.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("other", "legacy_value", "A"); + } + }); + } + + @Test + public void testStrictInvalidateAllFeedbackMakesNextBatchLookUpAndSurfaceFailure() throws Exception { + assertMemoryLeak(() -> { + int[] results = { + QwpSchemaProtocol.RESULT_DENIED, + QwpSchemaProtocol.RESULT_UNAVAILABLE, + QwpSchemaProtocol.RESULT_TOO_LARGE + }; + LineSenderSchemaException.Reason[] reasons = { + LineSenderSchemaException.Reason.ACCESS_DENIED, + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE + }; + for (int i = 0; i < results.length; i++) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_MISSING, -1, -1); + handler.isSchemaFeedbackEnabled = true; + // STRICT surfaces every lookup failure; AUTO would write the + // RESULT_UNAVAILABLE and RESULT_TOO_LARGE cases with the legacy contract instead. + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").longColumn("id", 1).atNow(); + handler.result = results[i]; + // The pending batch never consults the server: the conflict is + // local and only the partial row rolls back. + try { + sender.stringColumn("failed_only", "partial"); + sender.stringColumn("id", "different inferred type"); + Assert.fail("expected inferred type conflict"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + sender.longColumn("id", 3).atNow(); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", -1, -1, 2, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + reader.eof(); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The server could not describe the table for the stale frame, so + // its ACK invalidated the cache. The next explicit table selection + // looks the table up and reports the failure there. + try { + sender.table("events"); + Assert.fail("expected lookup failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reasons[i], e.getReason()); + } + Assert.assertEquals(2, handler.describeRequests.get()); + handler.result = QwpSchemaProtocol.RESULT_MISSING; + sender.table("events").longColumn("id", 5).atNow(); + sender.flush(); + reader = new FrameReader(handler.awaitDataFrames(2).get(1)); + reader.schemaTable("events", -1, -1, 1, "id", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(5, reader.i64()); + reader.eof(); + Assert.assertEquals(3, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testTableNegotiatesBeforeNullNoops() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").doubleArray("values", (double[]) null); + sender.longArray("values", (long[]) null); + sender.longArray("values", (long[][]) null); + sender.longArray("values", (long[][][]) null); + sender.longArray("values", (LongArray) null); + sender.ipv4Column("address", (CharSequence) null); + Assert.assertEquals(1, handler.describeRequests.get()); + sender.cancelRow(); + } + }); + } + + @Test + public void testUnsupportedSetterEvictsSnapshotAndInvalidNameDoesNotRefresh() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 2, 3); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events"); + try { + sender.stringColumn("bad\nname", "x"); + Assert.fail("expected invalid column name"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("column name")); + } + Assert.assertEquals(1, handler.describeRequests.get()); + sender.table("events"); + try { + sender.longColumn("id", 1); + Assert.fail("expected UUID target rejection"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, e.getReason()); + } + Assert.assertEquals("a rejected setter must not look up metadata", 1, + handler.describeRequests.get()); + // The snapshot type of a known column rejected the value, which a type + // change could explain, so the rejection evicted the snapshot and the + // next row looks the table up again. + sender.table("events").uuidColumn("id", 1, 2).atNow(); + Assert.assertEquals(2, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyFrameAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + for (boolean isStrict : new boolean[]{false, true}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + (isStrict ? "strict" : "auto") + ";close_flush_timeout_millis=0;")) { + sender.table("events").stringColumn("value", "legacy").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + FrameReader reader = new FrameReader(frame); + reader.legacyVarcharTable("events", "value", "legacy"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + } + }); + } + + @Test + public void testOldPeerUsesLegacySmallIntegerWireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events") + .byteColumn("b", (byte) -7) + .shortColumn("s", (short) 42) + .intColumn("i", Integer.MIN_VALUE) + .atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacySmallIntegerTable("events"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyIpv4WireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").ipv4Column("value", 0xc0a80101).atNow(); + sender.table("events").ipv4Column("value", "10.20.30.40").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacyIpv4Table( + "events", "value", 0xc0a80101, 0x0a141e28); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyLong256WireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").long256Column("value", 1, 2, 3, 4).atNow(); + sender.table("events").long256Column( + "value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE).atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacyLong256Table("events", "value"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testOldPeerUsesLegacyGeoHashWireAndNeverDescribes() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + try (TestWebSocketServer server = legacyServer(handler); + Sender sender = sender(server)) { + sender.table("events").geoHashColumn("value", 3, 5).atNow(); + sender.table("events").geoHashColumn("value", "U").atNow(); + sender.flush(); + byte[] frame = handler.awaitDataFrame(); + Assert.assertEquals(0, frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + new FrameReader(frame).legacyGeoHashTable("events", "value"); + Assert.assertEquals(0, handler.describeRequests.get()); + } + }); + } + + @Test + public void testVersionChangeDoesNotSplitPendingBatch() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 51, 52); + handler.isSchemaFeedbackEnabled = true; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.version = 53; + sender.table("events"); + try { + sender.stringColumn("id", "still-invalid"); + Assert.fail("expected UUID validation error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + // The rejection neither looks up metadata nor splits the batch: the + // next row joins the block pinned to version 52. + sender.uuidColumn("id", 3, 4).atNow(); + long fsn = sender.flushAndGetSequence(); + new FrameReader(handler.awaitDataFrame()).uuidTable("events", 51, 52, 1, 2, 3, 4); + Assert.assertEquals(1, handler.describeRequests.get()); + + // The stale frame's ACK carries version 53; the next batch adopts it. + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2).get(1)).uuidTable("events", 51, 53, 5, 6); + Assert.assertEquals("the ACK feedback replaces a second lookup", 1, + handler.describeRequests.get()); + } + }); + } + + @Test + public void testPendingLegacyBatchStaysLegacyAcrossUpgradeAndNextBatchAdoptsSchema() throws Exception { + assertMemoryLeak(() -> { + for (boolean isStrict : new boolean[]{false, true}) { + int port = TestPorts.findUnusedPort(); + SchemaHandler legacyHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 1, 1); + TestWebSocketServer legacy = new TestWebSocketServer(legacyHandler, false, null, port); + legacy.start(); + Assert.assertTrue(legacy.awaitStart(5, TimeUnit.SECONDS)); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = Sender.builder("ws::addr=localhost:" + port + + ";schema_mode=" + (isStrict ? "strict" : "auto") + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;") + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build(); + TestWebSocketServer upgraded = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + legacy.close(); + Assert.assertTrue("sender did not observe the legacy connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + sender.table("events").stringColumn("legacy_value", "B"); + SchemaHandler schemaHandler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 61, 62); + upgraded = new TestWebSocketServer(schemaHandler, false, null, port); + upgraded.setAdvertiseSchema(true); + upgraded.start(); + Assert.assertTrue(upgraded.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed and servicing frames", + schemaHandler.awaitPong()); + sender.atNow(); + // A batch has one wire contract: the pending batch already holds + // legacy rows, so this row stays legacy although the supporting + // connection is up. The first row after the flush adopts schema mode. + sender.table("events").stringColumn("legacy_value", "C").atNow(); + sender.flush(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = schemaHandler.awaitDataFrames(2); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(0, frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + Assert.assertTrue((frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + new FrameReader(frames.get(0)).legacyVarcharTable("events", "legacy_value", "A", "B", "C"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("events", 61, 62, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, schemaHandler.describeRequests.get()); + } finally { + sender.close(); + legacy.close(); + if (upgraded != null) { + upgraded.close(); + } + } + } + }); + } + + @Test(timeout = 10_000) + public void testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); + TestWebSocketServer server = null; + try { + Instant value = Instant.ofEpochSecond(3, 456_789_123); + // No server is listening: this effective write must nevertheless + // complete on the caller thread using the existing legacy conversion. + sender.table("events").timestampColumn("ts", value).atNow(); + + SchemaHandler handler = new SchemaHandler( + QwpSchemaProtocol.RESULT_KNOWN, 71, 72, true, -1); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + + // The pending batch is legacy, so this row stays legacy too. The + // first row after the flush adopts schema mode. + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); + sender.table("events").timestampColumn("ts", value).atNow(); + sender.flush(); + + List frames = handler.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyTimestampTable( + "events", "ts", 3_456_789L, 3_456_789L); + new FrameReader(frames.get(1)).schemaTimestampTable( + "events", 71, 72, "ts", 3_456_789_123L); + Assert.assertEquals(1, handler.describeRequests.get()); + } finally { + sender.close(); + if (server != null) { + server.close(); + } + } + }); + } + + @Test + public void testSchemaBatchCapSplitsWithoutDroppingPinnedFlags() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 81, 82); + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + server.setAdvertisedMaxBatchSize(64); + try (TestWebSocketServer ignored = start(server); + Sender sender = sender(server)) { + sender.table("a").uuidColumn("id", 1, 2).atNow(); + sender.table("b").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + List frames = handler.awaitDataFrames(2); + for (int i = 0; i < 2; i++) { + byte[] frame = frames.get(i); + Assert.assertTrue(frame.length <= 64); + Assert.assertTrue((frame[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, + ByteBuffer.wrap(frame).order(ByteOrder.LITTLE_ENDIAN).getShort(6) & 0xffff); + } + Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT) != 0); + Assert.assertEquals(0, frames.get(1)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_DEFER_COMMIT); + FrameReader first = new FrameReader(frames.get(0)); + first.schemaTable("a", 81, 82, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, first.u8()); + Assert.assertEquals(1, first.i64()); + Assert.assertEquals(2, first.i64()); + first.eof(); + FrameReader second = new FrameReader(frames.get(1)); + second.schemaTable("b", 82, 82, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, second.u8()); + Assert.assertEquals(3, second.i64()); + Assert.assertEquals(4, second.i64()); + second.eof(); + } + }); + } + + @Test(timeout = 10_000) + public void testUnboundedSchemaWaitStillResolves() throws Exception { + assertMemoryLeak(() -> { + for (String mode : new String[]{"strict", "auto"}) { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + // Long.MAX_VALUE once overflowed the absolute deadline into a zero + // wait: STRICT failed the row and AUTO fell back to legacy framing. + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=" + mode + ";schema_wait_millis=" + Long.MAX_VALUE + ";" + + "auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertEquals(mode, 1, handler.describeRequests.get()); + Assert.assertTrue(mode, (handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictSchemaLookupDeadlineFailsTypedThenRetryUsesAvailableSchema() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 91, 92); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;schema_wait_millis=1000;" + + "initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;auto_flush_rows=2147483647;" + + "auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + QwpTableBuffer b = ((QwpWebSocketSender) sender).getTableBuffer("b"); + sender.table("events").uuidColumn("id", 1, 2).atNow(); + QwpTableBuffer events = ((QwpWebSocketSender) sender).getTableBuffer("events"); + handler.respondToDescribe = false; + try { + sender.table("b"); + Assert.fail("expected schema negotiation deadline"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + Assert.assertEquals(1, events.getRowCount()); + Assert.assertEquals(0, b.getRowCount()); + handler.respondToDescribe = true; + sender.uuidColumn("id", 3, 4).atNow(); + Assert.assertEquals(1, events.getRowCount()); + Assert.assertEquals(1, b.getRowCount()); + sender.flush(); + Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + }); + } + + @Test + public void testStrictInterruptedSchemaLookupFailsTypedAndPreservesInterrupt() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 93, 94); + handler.respondToDescribe = false; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=strict;initial_connect_retry=async;reconnect_initial_backoff_millis=10;" + + "reconnect_max_backoff_millis=50;close_flush_timeout_millis=0;")) { + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch entered = new CountDownLatch(1); + AtomicReference writer = new AtomicReference<>(); + AtomicBoolean interruptPreserved = new AtomicBoolean(); + Future result = executor.submit(() -> { + writer.set(Thread.currentThread()); + entered.countDown(); + try { + sender.table("events").uuidColumn("id", 1, 2); + return null; + } catch (LineSenderSchemaException e) { + interruptPreserved.set(Thread.currentThread().isInterrupted()); + return e; + } + }); + try { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + writer.get().interrupt(); + LineSenderSchemaException error = result.get(5, TimeUnit.SECONDS); + Assert.assertNotNull(error); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, error.getReason()); + Assert.assertTrue("writer interrupt status was cleared", interruptPreserved.get()); + } finally { + executor.shutdownNow(); + Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS)); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictAsyncRecoveredSchemaBacklogWaitsForNegotiation() throws Exception { + assertMemoryLeak(() -> { + File sfRoot = temporaryFolder.newFolder("schema-required-recovery"); + int unavailablePort = publishUnackedSchemaBacklog(sfRoot, 95, 96); + + // STRICT never takes the legacy contract once schema support is known, + // and the recovered backlog already requires it. With no server the + // wait runs out its configured budget and fails typed. + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + unavailablePort + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;initial_connect_retry=async;schema_mode=strict;" + + "schema_wait_millis=250;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;")) { + try { + sender.table("events"); + Assert.fail("recovered schema-required backlog fell back to legacy encoding"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoAsyncRecoveredSchemaBacklogWritesLegacyOfflineAndReplaysBacklog() throws Exception { + assertMemoryLeak(() -> { + File sfRoot = temporaryFolder.newFolder("schema-required-recovery-auto"); + int port = publishUnackedSchemaBacklog(sfRoot, 97, 98); + + TestWebSocketServer server = null; + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;initial_connect_retry=async;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;")) { + // AUTO (the default) never waits on a wire that has not come up. The + // recovered backlog still pins the capability requirement for the + // reconnect, but this row takes the legacy contract right away. + sender.table("events").stringColumn("legacy_value", "A").atNow(); + + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 97, 98); + server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertTrue("schema-capable connection was not installed", handler.awaitPong()); + sender.flush(); + + // The recovered schema frame replays first, then the legacy batch. + List frames = handler.awaitDataFrames(2); + FrameReader recovered = new FrameReader(frames.get(0)); + recovered.schemaTable("events", 97, 98, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, recovered.u8()); + Assert.assertEquals(1, recovered.i64()); + Assert.assertEquals(2, recovered.i64()); + recovered.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("events", "legacy_value", "A"); + Assert.assertEquals(0, handler.describeRequests.get()); + } finally { + if (server != null) { + server.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoOutageWritesUnseenTableLegacyAndNextBatchAdoptsSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 111, 112); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((first.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // The wire is down and "other" is not cached. AUTO must not wait out + // the 30 s default schema budget (the test timeout proves it); the row + // takes the legacy contract and queues in store-and-forward. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 113, 114); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + // The first row after the flush consults the schema again: "other" is + // looked up on the fresh connection and adopts the strict contract. + sender.table("other").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + + List frames = second.awaitDataFrames(2); + new FrameReader(frames.get(0)).legacyVarcharTable("other", "legacy_value", "A"); + FrameReader schema = new FrameReader(frames.get(1)); + schema.schemaTable("other", 113, 114, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(3, schema.i64()); + Assert.assertEquals(4, schema.i64()); + schema.eof(); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testReconnectKeepsCachedSchema() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + new FrameReader(first.awaitDataFrame()).schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(1, first.describeRequests.get()); + Assert.assertTrue(sender.awaitAckedFsn(sender.flushAndGetSequence(), 5_000)); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + // The next batch binds from the snapshot cached before the outage, so + // the reconnect costs no DESCRIBE round trip. + sender.table("events").uuidColumn("id", 3, 4).atNow(); + sender.flush(); + FrameReader frame = new FrameReader(second.awaitDataFrame()); + frame.schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, frame.u8()); + Assert.assertEquals(3, frame.i64()); + Assert.assertEquals(4, frame.i64()); + frame.eof(); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testAutoOutagePublishesPendingSchemaBatchBeforeLegacyRow() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer restarted = null; + try { + // A schema row is pending and unflushed when the wire drops. + sender.table("events").uuidColumn("id", 1, 2).atNow(); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // A QWP frame has one wire family: the legacy row cannot join the + // pending schema batch, so that batch is published to store-and-forward + // first and the legacy row opens a new batch. + sender.table("other").stringColumn("legacy_value", "A").atNow(); + Assert.assertEquals(0, ((QwpWebSocketSender) sender).getTableBuffer("events").getRowCount()); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 121, 122); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + + List frames = second.awaitDataFrames(2); + FrameReader schema = new FrameReader(frames.get(0)); + schema.schemaTable("events", 121, 122, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, schema.u8()); + Assert.assertEquals(1, schema.i64()); + Assert.assertEquals(2, schema.i64()); + schema.eof(); + new FrameReader(frames.get(1)).legacyVarcharTable("other", "legacy_value", "A"); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testStrictOutageFailsUnseenTableWithinSchemaWaitAndRecovers() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 131, 132); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=strict;schema_wait_millis=250;", disconnected); + TestWebSocketServer restarted = null; + try { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((first.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + + // STRICT keeps waiting for the wire and fails typed at the budget. + try { + sender.table("other"); + Assert.fail("STRICT fell back to legacy encoding during an outage"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, e.getReason()); + } + // A cached table is still writable offline: "events" was pinned by the + // flushed batch's snapshot and the cache survives until reconnect. + sender.table("events").uuidColumn("id", 3, 4).atNow(); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 131, 132); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.table("other").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + + List frames = second.awaitDataFrames(1); + Assert.assertTrue((frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(2, ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN) + .getShort(6) & 0xffff); + Assert.assertEquals(1, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testOffRetainedLegacyFramesFailOverFromCapableToLegacyPeer() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142, false); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=off;", disconnected); + TestWebSocketServer replacement = null; + try { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + byte[] retained = first.awaitDataFrame(); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertArrayEquals(retained, second.awaitDataFrame()); + Assert.assertTrue(sender.drain(5_000)); + Assert.assertFalse(server.hasRequestedSchema()); + Assert.assertFalse(replacement.hasRequestedSchema()); + Assert.assertEquals(0, first.describeRequests.get()); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testStrictRemembersSchemaHandshakeWithoutPublishedFrames() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "schema_mode=strict;schema_wait_millis=100;", disconnected); + TestWebSocketServer replacement = null; + try { + Assert.assertTrue(first.awaitPong()); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + LineSenderSchemaException outage = Assert.assertThrows(LineSenderSchemaException.class, + () -> sender.table("events")); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, outage.getReason()); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertTrue("legacy replacement was not installed", second.awaitPong()); + LineSenderSchemaException downgrade = Assert.assertThrows(LineSenderSchemaException.class, + () -> sender.table("other")); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, downgrade.getReason()); + Assert.assertEquals(0, first.describeRequests.get()); + Assert.assertEquals(0, second.describeRequests.get()); + Assert.assertEquals(0, second.dataFrames.size()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 15_000) + public void testAutoDowngradesBeforePublishingSchemaFrames() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + Sender sender = outageSender(port, "", disconnected); + TestWebSocketServer replacement = null; + try { + // Lookup alone must not turn a schema-capable peer into a replay requirement. + sender.table("events"); + Assert.assertEquals(1, first.describeRequests.get()); + server.close(); + Assert.assertTrue(disconnected.await(5, TimeUnit.SECONDS)); + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + replacement = start(new TestWebSocketServer(second, false, null, port)); + Assert.assertTrue("legacy replacement was not installed", second.awaitPong()); + sender.table("other").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + new FrameReader(second.awaitDataFrame()).legacyVarcharTable("other", "legacy_value", "A"); + Assert.assertEquals(0, second.describeRequests.get()); + } finally { + sender.close(); + server.close(); + if (replacement != null) { + replacement.close(); + } + } + }); + } + + @Test(timeout = 10_000) + public void testSchemaModeOffNeverDescribesOnSchemaCapableServer() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 141, 142); + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;auto_flush_rows=2147483647;auto_flush_bytes=0;" + + "auto_flush_interval=2147483646;close_flush_timeout_millis=0;")) { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); + Assert.assertEquals(0, handler.describeRequests.get()); + Assert.assertFalse(server.hasRequestedSchema()); + } + }); + } + + @Test(timeout = 10_000) + public void testAutoAbandonsLookupWhenWireDropsUnderIt() throws Exception { + assertMemoryLeak(() -> { + int port = TestPorts.findUnusedPort(); + SchemaHandler first = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 151, 152); + first.respondToDescribe = false; + TestWebSocketServer server = schemaServer(first, port); + CountDownLatch disconnected = new CountDownLatch(1); + // A budget far beyond the test timeout: only the wire drop can end the wait. + Sender sender = outageSender(port, "schema_wait_millis=60000;", disconnected); + ExecutorService executor = Executors.newSingleThreadExecutor(); + TestWebSocketServer restarted = null; + try { + Future row = executor.submit(() -> { + sender.table("events").stringColumn("legacy_value", "A").atNow(); + return null; + }); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (first.describeRequests.get() == 0) { + Assert.assertTrue("lookup was never sent", System.nanoTime() < deadline); + Thread.sleep(5); + } + server.close(); + Assert.assertTrue("sender did not observe the connection closing", + disconnected.await(5, TimeUnit.SECONDS)); + // The connection loss fails the pending lookup and AUTO falls back to + // the legacy contract instead of sitting out the schema budget. + row.get(5, TimeUnit.SECONDS); + + SchemaHandler second = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 151, 152); + restarted = schemaServer(second, port); + Assert.assertTrue("schema-capable connection was not installed", second.awaitPong()); + sender.flush(); + new FrameReader(second.awaitDataFrame()).legacyVarcharTable("events", "legacy_value", "A"); + } finally { + executor.shutdownNow(); + sender.close(); + server.close(); + if (restarted != null) { + restarted.close(); + } + } + }); + } + + @Test + public void testResetDiscardsRowsAndKeepsCachedSnapshot() throws Exception { + assertMemoryLeak(() -> { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, 101, 102); + handler.isSchemaFeedbackEnabled = true; + try (TestWebSocketServer server = schemaServer(handler); + Sender sender = sender(server)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + handler.version = 103; + sender.table("events"); + try { + sender.stringColumn("id", "invalid"); + Assert.fail("expected UUID validation error"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + // reset() discards the rows without sending anything, so no ACK can + // deliver version 103: the next batch still pins the cached 102. + sender.reset(); + sender.table("events").uuidColumn("id", 3, 4).atNow(); + long fsn = sender.flushAndGetSequence(); + FrameReader reader = new FrameReader(handler.awaitDataFrame()); + reader.schemaTable("events", 101, 102, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(4, reader.i64()); + reader.eof(); + // That frame's ACK carries 103 for the batch after it. + Assert.assertTrue(sender.awaitAckedFsn(fsn, 5_000)); + sender.table("events").uuidColumn("id", 5, 6).atNow(); + sender.flush(); + new FrameReader(handler.awaitDataFrames(2).get(1)).uuidTable("events", 101, 103, 5, 6); + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + @Test + public void testNormalSenderAutomaticallyNegotiatesBeforeFirstEffectiveValue() throws Exception { + assertMemoryLeak(() -> { + DescribeUuidHandler handler = new DescribeUuidHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + sender.table("events"); + try { + sender.stringColumn("id", "not-a-uuid"); + sender.cancelRow(); + Assert.fail("expected schema-directed UUID validation"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + } + } + Assert.assertEquals(1, handler.describeRequests.get()); + } + }); + } + + private static final class DescribeUuidHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger describeRequests = new AtomicInteger(); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + ByteBuffer input = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + if (input.getInt(0) != QwpConstants.MAGIC_MESSAGE + || data[5] != QwpSchemaProtocol.FLAG_CONTROL) { + return; + } + describeRequests.incrementAndGet(); + long requestId = input.getLong(QwpConstants.HEADER_SIZE + 1); + byte[] column = "id".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + ByteBuffer response = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + response.putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(17) + .putLong(23) + .putShort((short) -1) + .putShort((short) 1) + .putShort((short) column.length) + .put(column) + .putInt(ColumnType.UUID) + .putShort((short) 0); + try { + client.sendBinary(response.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private static TestWebSocketServer legacyServer(SchemaHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + /** + * A sender for the outage tests: no auto-flush, fast reconnect backoff, and a + * connection listener that trips {@code disconnected} when the wire drops. + * {@code extraConfig} is appended verbatim (e.g. {@code schema_mode=strict;}). + */ + private static Sender outageSender(int port, String extraConfig, CountDownLatch disconnected) { + return Sender.builder("ws::addr=localhost:" + port + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;" + + "close_flush_timeout_millis=0;" + extraConfig) + .connectionListener(event -> { + if (event.getKind() == SenderConnectionEvent.Kind.DISCONNECTED) { + disconnected.countDown(); + } + }) + .build(); + } + + /** + * Writes one schema-encoded UUID row for "events" through a sender rooted at + * {@code sfRoot} against a server that never acknowledges data, so the frame + * stays in store-and-forward for a later sender to recover. Returns the port + * the server listened on, which is free again when this method returns. + */ + private static int publishUnackedSchemaBacklog(File sfRoot, int tableId, long version) throws Exception { + SchemaHandler handler = new SchemaHandler(QwpSchemaProtocol.RESULT_KNOWN, tableId, version, false); + try (TestWebSocketServer server = schemaServer(handler)) { + String cfg = "ws::addr=localhost:" + server.getPort() + + ";sf_dir=" + sfRoot.getAbsolutePath() + + ";sender_id=recovery;close_flush_timeout_millis=0;"; + try (Sender sender = Sender.fromConfig(cfg)) { + sender.table("events").uuidColumn("id", 1, 2).atNow(); + sender.flush(); + Assert.assertTrue((handler.awaitDataFrame()[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA) != 0); + } + return server.getPort(); + } + } + + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + return start(server); + } + + private static TestWebSocketServer schemaServer(TestWebSocketServer.WebSocketServerHandler handler, int port) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler, false, null, port); + server.setAdvertiseSchema(true); + return start(server); + } + + private static TestWebSocketServer start(TestWebSocketServer server) throws Exception { + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + private static byte numericWireType(int targetType) { + switch (targetType) { + case ColumnType.BYTE: + return QwpConstants.TYPE_BYTE; + case ColumnType.SHORT: + return QwpConstants.TYPE_SHORT; + case ColumnType.INT: + return QwpConstants.TYPE_INT; + case ColumnType.LONG: + return QwpConstants.TYPE_LONG; + case ColumnType.FLOAT: + return QwpConstants.TYPE_FLOAT; + case ColumnType.DOUBLE: + return QwpConstants.TYPE_DOUBLE; + default: + throw new AssertionError(ColumnType.nameOf(targetType)); + } + } + + private static byte temporalWireType(int targetType) { + switch (targetType) { + case ColumnType.DATE: + return QwpConstants.TYPE_DATE; + case ColumnType.TIMESTAMP_MICRO: + return QwpConstants.TYPE_TIMESTAMP; + case ColumnType.TIMESTAMP_NANO: + return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: + throw new AssertionError(ColumnType.nameOf(targetType)); + } + } + + private static Sender sender(TestWebSocketServer server) { + return Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";auto_flush_rows=2147483647;auto_flush_bytes=0;auto_flush_interval=2147483646;" + + "close_flush_timeout_millis=0;"); + } + + private static final class DecimalTextVector { + private final String caseId; + private final String expectedText; + private final String inputForm; + private final String sourceType; + private final int sourceScale; + private final String[] limbs; + private final String targetType; + + private DecimalTextVector(String[] fields) { + Assert.assertEquals(String.join("\t", fields), 10, fields.length); + caseId = fields[0]; + sourceType = fields[1]; + sourceScale = Integer.parseInt(fields[2]); + limbs = new String[]{fields[3], fields[4], fields[5], fields[6]}; + targetType = fields[7]; + inputForm = fields[8]; + expectedText = fields[9]; + } + + private void append(Sender sender, String column) { + if ("JAVA_NULL".equals(inputForm)) { + appendJavaNull(sender, column); + return; + } + if ("NULL_VALUE".equals(inputForm)) { + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, Decimal64.NULL_VALUE); + return; + case "DECIMAL128": + sender.decimalColumn(column, Decimal128.NULL_VALUE); + return; + case "DECIMAL256": + sender.decimalColumn(column, Decimal256.NULL_VALUE); + return; + default: + throw new AssertionError(sourceType); + } + } + Assert.assertEquals("VALUE", inputForm); + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, new Decimal64(limb(0), sourceScale)); + return; + case "DECIMAL128": + sender.decimalColumn(column, new Decimal128(limb(1), limb(0), sourceScale)); + return; + case "DECIMAL256": + sender.decimalColumn(column, + new Decimal256(limb(3), limb(2), limb(1), limb(0), sourceScale)); + return; + default: + throw new AssertionError(sourceType); + } + } + + private void appendJavaNull(Sender sender, String column) { + switch (sourceType) { + case "DECIMAL64": + sender.decimalColumn(column, (Decimal64) null); + return; + case "DECIMAL128": + sender.decimalColumn(column, (Decimal128) null); + return; + case "DECIMAL256": + sender.decimalColumn(column, (Decimal256) null); + return; + default: + throw new AssertionError(sourceType); + } + } + + private boolean isNull() { + return !"VALUE".equals(inputForm); + } + + private long limb(int index) { + Assert.assertNotEquals("-", limbs[index]); + return Long.parseUnsignedLong(limbs[index], 16); + } + + private int sourceIndex() { + switch (sourceType) { + case "DECIMAL64": + return 0; + case "DECIMAL128": + return 1; + case "DECIMAL256": + return 2; + default: + throw new AssertionError(sourceType); + } + } + + private int targetIndex() { + if ("STRING".equals(targetType)) { + return 0; + } + if ("VARCHAR".equals(targetType)) { + return 1; + } + throw new AssertionError(targetType); + } + + private int targetType() { + if ("STRING".equals(targetType)) { + return ColumnType.STRING; + } + if ("VARCHAR".equals(targetType)) { + return ColumnType.VARCHAR; + } + throw new AssertionError(targetType); + } + } + + /** + * Builds the cumulative ACK the fake servers send for a data frame. Like the real + * server, a handler attaches the current schema of {@code tableName} when the + * snapshot it last delivered to the client is stale, so the client learns about + * a change on the batch boundary without a DESCRIBE of its own. + */ + private static byte[] ackWithSchemaUpdate(long sequence, String tableName, byte[] schemaPayload) { + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + ByteBuffer ack = ByteBuffer.allocate(1 + 8 + 2 + 2 + 2 + name.length + 4 + schemaPayload.length) + .order(ByteOrder.LITTLE_ENDIAN); + ack.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(sequence).putShort((short) 0) + .putShort((short) 1).putShort((short) name.length).put(name) + .putInt(schemaPayload.length).put(schemaPayload); + return ack.array(); + } + + /** The ACK a server sends when it cannot produce the schema snapshot a stale frame calls for. */ + private static byte[] ackInvalidateAll(long sequence) { + ByteBuffer ack = ByteBuffer.allocate(1 + 8 + 2).order(ByteOrder.LITTLE_ENDIAN); + ack.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)) + .putLong(sequence).putShort((short) 0); + return ack.array(); + } + + private static byte[] schemaControlFrame(byte[] schemaPayload) { + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + schemaPayload.length) + .order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL).putShort((short) 0).putInt(schemaPayload.length) + .put(schemaPayload); + return out.array(); + } + + /** + * Walks one QWP data frame, or a sequence of single-table frames. A table + * pins one schema snapshot per batch, so two schema generations of the same + * table always arrive in two frames; the two-block helpers read the second + * block from the second frame, skipping its message header and symbol + * dictionary prelude. + */ + private static final class FrameReader { + private final List frames; + private int frameIndex; + private ByteBuffer in; + + private FrameReader(byte[] data) { + this(Collections.singletonList(data)); + } + + private FrameReader(List frames) { + this.frames = frames; + this.in = ByteBuffer.wrap(frames.get(0)).order(ByteOrder.LITTLE_ENDIAN); + } + + private void nextFrameIfExhausted() { + if (in.hasRemaining() || frameIndex + 1 >= frames.size()) { + return; + } + frameIndex++; + in = ByteBuffer.wrap(frames.get(frameIndex)).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + varint(); + for (int i = 0, n = varint(); i < n; i++) { + string(); + } + } + + private long i64() { + return in.getLong(); + } + + private void legacyVarcharTable(String table, String column, String... values) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(values.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + int total = 0; + StringBuilder joined = new StringBuilder(); + for (String value : values) { + joined.append(value); + total += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(total, in.getInt()); + } + byte[] bytes = joined.toString().getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[total]; + in.get(actual); + Assert.assertArrayEquals(bytes, actual); + eof(); + } + + private void legacyTimestampTable(String table, String column, long... expectedMicros) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(expectedMicros.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + for (long micros : expectedMicros) { + Assert.assertEquals(micros, i64()); + } + eof(); + } + + private void schemaTimestampTable( + String table, + int tableId, + long version, + String column, + long expectedNanos + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 1, column, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedNanos, i64()); + eof(); + } + + private void legacyIpv4Table(String table, String column, int... values) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(values.length, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_IPv4, u8()); + Assert.assertEquals(0, u8()); + for (int value : values) { + Assert.assertEquals(value, in.getInt()); + } + eof(); + } + + private void legacyLong256Table(String table, String column) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, u8()); + Assert.assertEquals(0, u8()); + for (long value : new long[]{1, 2, 3, 4}) { + Assert.assertEquals(value, i64()); + } + for (int i = 0; i < 4; i++) { + Assert.assertEquals(Long.MIN_VALUE, i64()); + } + eof(); + } + + private void legacySmallIntegerTable(String table) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(3, varint()); + Assert.assertEquals("b", string()); + Assert.assertEquals(QwpConstants.TYPE_BYTE, u8()); + Assert.assertEquals("s", string()); + Assert.assertEquals(QwpConstants.TYPE_SHORT, u8()); + Assert.assertEquals("i", string()); + Assert.assertEquals(QwpConstants.TYPE_INT, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals((byte) -7 & 0xff, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(42, in.getShort()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Integer.MIN_VALUE, in.getInt()); + eof(); + } + + private void longTextTable( + String table, + int tableId, + long version, + int targetType, + String expectedValue + ) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + // Global dictionary registration is append-only: the rolled-back B value + // remains an unused dictionary entry, while no B row survives. + Assert.assertEquals("7", string()); + Assert.assertEquals(expectedValue, string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, tableId, version, 2, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(1, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + } + eof(); + } + + private void ipv4Table(String table, int tableId, long version, int targetType, int expectedValue) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = targetType == ColumnType.IPv4 + ? QwpConstants.TYPE_IPv4 + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, tableId, version, 2, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (wireType == QwpConstants.TYPE_IPv4) { + Assert.assertEquals(expectedValue, in.getInt()); + } else { + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(9, in.getInt()); + Assert.assertEquals("255.1.2.3", stringBytes(9)); + } + eof(); + } + + private void intToIpv4Table(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, u8()); + Assert.assertEquals(3, u8()); + Assert.assertEquals(1, in.getInt()); + Assert.assertEquals(-1, in.getInt()); + eof(); + } + + private void long256TargetTable( + String table, + int tableId, + long version, + int targetType, + long l0, + long l1, + long l2, + long l3 + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = targetType == ColumnType.LONG256 + ? QwpConstants.TYPE_LONG256 + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, tableId, version, 2, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (wireType == QwpConstants.TYPE_LONG256) { + Assert.assertEquals(l0, i64()); + Assert.assertEquals(l1, i64()); + Assert.assertEquals(l2, i64()); + Assert.assertEquals(l3, i64()); + } else { + String expected = "0x04000000000000000300000000000000020000000000000001"; + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + } + eof(); + } + + private void long256Table(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0xfedcba9876543210L, i64()); + Assert.assertEquals(0x0123456789abcdefL, i64()); + Assert.assertEquals(0, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void geoHashTable( + String table, int tableId, long version, int bits, int first, int second + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(bits, varint()); + Assert.assertEquals(first, u8()); + Assert.assertEquals(second, u8()); + eof(); + } + + private void geoHashTargetTable( + String table, + int tableId, + long version, + int targetType, + long first, + long second, + int precision + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + byte wireType = ColumnType.isGeoHash(targetType) + ? QwpConstants.TYPE_GEOHASH + : QwpConstants.TYPE_VARCHAR; + schemaBlockHeader(table, tableId, version, 2, "value", wireType); + if (wireType == QwpConstants.TYPE_GEOHASH) { + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(precision, varint()); + for (long value : new long[]{first, second}) { + for (int i = 0; i < (precision + 7) / 8; i++) { + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), u8()); + } + } + } else { + String firstText = "11010000110001101100"; + String secondText = "00010010001101000101"; + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(firstText.length(), in.getInt()); + Assert.assertEquals(firstText.length() + secondText.length(), in.getInt()); + Assert.assertEquals(firstText, stringBytes(firstText.length())); + Assert.assertEquals(secondText, stringBytes(secondText.length())); + } + eof(); + } + + private void legacyGeoHashTable(String table, String column) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertEquals(0, u8() & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, varint()); + Assert.assertEquals(3, u8()); + Assert.assertEquals(26, u8()); + eof(); + } + + private void decimal64Table( + String table, int tableId, long version, int scale, long first, long second + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(scale, u8()); + Assert.assertEquals(first, i64()); + Assert.assertEquals(second, i64()); + eof(); + } + + private void decimalTextLifecycleTable(String table, int tableId, long version) { + StringBuilder maxScale = new StringBuilder(78).append("0."); + for (int i = 0; i < 76; i++) { + maxScale.append('9'); + } + String expected = "123.40-0.125" + maxScale + '0'; + + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 5, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0x04, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals(12, in.getInt()); + Assert.assertEquals(90, in.getInt()); + Assert.assertEquals(91, in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void decimalTextTable(String table, int tableId, long version, String... expectedValues) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, expectedValues.length, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + int totalLength = 0; + StringBuilder expected = new StringBuilder(); + for (String value : expectedValues) { + expected.append(value); + totalLength += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(totalLength, in.getInt()); + } + byte[] expectedBytes = expected.toString().getBytes(StandardCharsets.UTF_8); + byte[] actual = new byte[totalLength]; + in.get(actual); + Assert.assertArrayEquals(expectedBytes, actual); + eof(); + } + + private void parsedDecimalTextTable(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0x0c, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals(12, in.getInt()); + Assert.assertEquals("123.451.2300", stringBytes(12)); + eof(); + } + + private void twoDecimalBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1_000, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void twoDecimal64ScaleBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(12_345, i64()); + eof(); + } + + private void twoDecimal64AndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(12_340, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(7, in.getInt()); + Assert.assertEquals("25.0000", stringBytes(7)); + eof(); + } + + private void twoDecimalWidthAndScaleBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(10_000, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, u8()); + Assert.assertEquals(-8_446_744_073_709_551_616L, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void nullGeoHashTable(String table, int tableId, long version, int bits) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(bits, varint()); + eof(); + } + + private void smallIntegerNumericTable(String table, int tableId, long version, byte wireType) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + assertSmallIntegerValue(wireType, -1); + assertSmallIntegerValue(wireType, 2); + assertSmallIntegerValue(wireType, 3); + eof(); + } + + private void smallIntegerDecimalTable(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 4, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + Assert.assertEquals(2, u8()); + Assert.assertEquals(4_200, i64()); + Assert.assertEquals(-700, i64()); + Assert.assertEquals(214_748_364_700L, i64()); + eof(); + } + + private void smallIntegerTextTable(String table, int tableId, long version, int targetType) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(4, varint()); + Assert.assertEquals("-128", string()); + Assert.assertEquals("99", string()); + Assert.assertEquals("32767", string()); + Assert.assertEquals("2147483647", string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, tableId, version, 4, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(3, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(4, in.getInt()); + Assert.assertEquals(9, in.getInt()); + Assert.assertEquals(19, in.getInt()); + Assert.assertEquals("-128327672147483647", stringBytes(19)); + } + eof(); + } + + private void integerTemporalTable(String table, int tableId, long version, byte wireType) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 5, "value", wireType); + Assert.assertEquals(1, u8()); + Assert.assertEquals(4, u8()); + if (wireType != QwpConstants.TYPE_DATE) { + Assert.assertEquals(0, u8()); + } + Assert.assertEquals(-1, i64()); + Assert.assertEquals(2, i64()); + Assert.assertEquals(3, i64()); + Assert.assertEquals(Long.MAX_VALUE, i64()); + eof(); + } + + private void assertSmallIntegerValue(byte wireType, long value) { + switch (wireType) { + case QwpConstants.TYPE_BYTE: + Assert.assertEquals((byte) value & 0xff, u8()); + break; + case QwpConstants.TYPE_SHORT: + Assert.assertEquals((short) value, in.getShort()); + break; + case QwpConstants.TYPE_INT: + Assert.assertEquals((int) value, in.getInt()); + break; + case QwpConstants.TYPE_LONG: + Assert.assertEquals(value, in.getLong()); + break; + case QwpConstants.TYPE_FLOAT: + Assert.assertEquals(Float.floatToRawIntBits((float) value), in.getInt()); + break; + case QwpConstants.TYPE_DOUBLE: + Assert.assertEquals(Double.doubleToRawLongBits((double) value), in.getLong()); + break; + default: + throw new AssertionError(wireType); + } + } + + private void twoGeoHashBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(60, varint()); + Assert.assertEquals(0x443214c74254bL, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, varint()); + Assert.assertEquals(31, u8()); + eof(); + } + + private void twoDateAndTimestampBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(12, i64()); + eof(); + } + + private void twoLong256AndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0xfedcba9876543210L, i64()); + Assert.assertEquals(0x0123456789abcdefL, i64()); + Assert.assertEquals(0, i64()); + Assert.assertEquals(0, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(11, in.getInt()); + Assert.assertEquals("not-long256", stringBytes(11)); + eof(); + } + + private void twoNativeLong256AndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_LONG256); + Assert.assertEquals(0, u8()); + for (long value : new long[]{1, 2, 3, 4}) { + Assert.assertEquals(value, i64()); + } + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + String expected = "0x08000000000000000700000000000000060000000000000005"; + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void twoNativeGeoHashAndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(20, varint()); + Assert.assertEquals(0xde, u8()); + Assert.assertEquals(0xbc, u8()); + Assert.assertEquals(0x0a, u8()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + String expected = "00010010001101000101"; + Assert.assertEquals(expected.length(), in.getInt()); + Assert.assertEquals(expected, stringBytes(expected.length())); + eof(); + } + + private void longTimestampTable( + String table, int tableId, long version, int targetType, long expectedValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", + targetType == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(expectedValue, i64()); + eof(); + } + + private void twoLongAndTimestampBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_LONG); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(12, i64()); + eof(); + } + + private void twoVarcharAndTimestampBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + eof(); + } + + private void timestampTextTable( + String table, int tableId, long version, String expectedLastValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedLastValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + eof(); + } + + private void stringDateTable( + String table, int tableId, long version, long expectedValue + ) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(expectedValue, i64()); + eof(); + } + + private void doubleArrayTable(String table, int tableId, long version) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{1, 2}, 1.0, 2.0); + assertDoubleArray(new int[]{2, 1}, 3.0, 4.0); + eof(); + } + + private void twoDoubleArrayRankBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{1, 2}, 1.0, 2.0); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DOUBLE_ARRAY); + Assert.assertEquals(0, u8()); + assertDoubleArray(new int[]{2}, 3.0, 4.0); + eof(); + } + + private void assertDoubleArray(int[] shape, double... values) { + Assert.assertEquals(shape.length, u8()); + for (int dimension : shape) { + Assert.assertEquals(dimension, in.getInt()); + } + for (double value : values) { + Assert.assertEquals(Double.doubleToRawLongBits(value), in.getLong()); + } + } + + private void twoDateAndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DATE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + eof(); + } + + private void charTable(String table, int tableId, long version, char first, char second) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, 2, "value", QwpConstants.TYPE_CHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(first, in.getChar()); + Assert.assertEquals(second, in.getChar()); + eof(); + } + + private void twoVarcharAndCharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(8, in.getInt()); + Assert.assertEquals("native-a", stringBytes(8)); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_CHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0xffff, in.getChar()); + eof(); + } + + private void twoTimestampAndVarcharBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Long.MIN_VALUE, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + String expected = "2262-04-11T23:47:16.854Z"; + byte[] bytes = expected.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(bytes.length, in.getInt()); + byte[] actual = new byte[bytes.length]; + in.get(actual); + Assert.assertArrayEquals(bytes, actual); + eof(); + } + + private void floatingTextTable( + String table, + int tableId, + long version, + int targetType, + String rolledBackValue, + String expectedValue + ) { + messageHeader(1); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(0, varint()); + Assert.assertEquals(2, varint()); + Assert.assertEquals(rolledBackValue, string()); + Assert.assertEquals(expectedValue, string()); + } else { + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + } + schemaBlockHeader(table, tableId, version, 2, "value", + targetType == ColumnType.SYMBOL ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, u8()); + Assert.assertEquals(1, u8()); + if (targetType == ColumnType.SYMBOL) { + Assert.assertEquals(1, varint()); + } else { + Assert.assertEquals(0, in.getInt()); + byte[] expected = expectedValue.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(expected.length, in.getInt()); + byte[] actual = new byte[expected.length]; + in.get(actual); + Assert.assertArrayEquals(expected, actual); + } + eof(); + } + + private void messageHeader(int tableCount) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + if (frames.size() == 1) { + Assert.assertEquals(tableCount, in.getShort() & 0xffff); + } else { + // One generation per frame: the blocks span the frame sequence. + Assert.assertEquals(tableCount, frames.size()); + Assert.assertEquals(1, in.getShort() & 0xffff); + } + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + } + + private void schemaBlockHeader( + String table, + int tableId, + long version, + int rows, + String column, + byte wireType + ) { + nextFrameIfExhausted(); + Assert.assertEquals(table, string()); + Assert.assertEquals(1, u8()); + Assert.assertEquals(tableId, in.getInt()); + Assert.assertEquals(version, in.getLong()); + Assert.assertEquals(rows, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(column, string()); + Assert.assertEquals(wireType & 0xff, u8()); + } + + private void twoSymbolAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("7", string()); + + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_SYMBOL); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(1, in.getInt()); + Assert.assertEquals('8', u8()); + eof(); + } + + private void twoSmallIntegerDecimalBlocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_DECIMAL128); + Assert.assertEquals(0, u8()); + Assert.assertEquals(5, u8()); + Assert.assertEquals(100_000_000, i64()); + Assert.assertEquals(0, i64()); + eof(); + } + + private void twoIntDecimalAndIpv4Blocks( + String table, int tableId, long firstVersion, long secondVersion + ) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1_000, in.getInt()); + eof(); + } + + private void twoUuidAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x0102030405060708L, i64()); + Assert.assertEquals(0x1112131415161718L, i64()); + + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(36, in.getInt()); + Assert.assertEquals("31323334-3536-3738-2122-232425262728", stringBytes(36)); + eof(); + } + + private void twoIpv4AndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x01020304, in.getInt()); + + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(7, in.getInt()); + Assert.assertEquals("5.6.7.8", stringBytes(7)); + eof(); + } + + private void twoDoubleAndVarcharBlocks(String table, int tableId, long firstVersion, long secondVersion) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, firstVersion, 1, "value", QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, u8()); + Assert.assertEquals(Double.doubleToRawLongBits(-0.0), i64()); + schemaBlockHeader(table, tableId, secondVersion, 1, "value", QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0, in.getInt()); + Assert.assertEquals(6, in.getInt()); + Assert.assertEquals("1.0E23", stringBytes(6)); + eof(); + } + + private String stringBytes(int length) { + byte[] bytes = new byte[length]; + in.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private void eof() { + Assert.assertEquals("unread frames", frames.size() - 1, frameIndex); + Assert.assertFalse("unexpected trailing column or value bytes", in.hasRemaining()); + } + + private String string() { + int length = varint(); + byte[] bytes = new byte[length]; + in.get(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private void schemaTable(String table, int tableId, long version, int rows, String column, byte type) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, in.getInt()); + Assert.assertEquals(QwpConstants.VERSION, u8()); + Assert.assertTrue((u8() & QwpConstants.FLAG_SCHEMA) != 0); + Assert.assertEquals(1, in.getShort() & 0xffff); + Assert.assertEquals(in.remaining() - Integer.BYTES, in.getInt()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + Assert.assertEquals(table, string()); + if (tableId < 0) { + Assert.assertEquals(0, u8()); + } else { + Assert.assertEquals(1, u8()); + Assert.assertEquals(tableId, in.getInt()); + Assert.assertEquals(version, in.getLong()); + } + Assert.assertEquals(rows, varint()); + int columns = varint(); + Assert.assertEquals(1, columns); + Assert.assertEquals(column, string()); + Assert.assertEquals(type & 0xff, u8()); + } + + private void uuidTable(String table, int tableId, long version, long... loHiPairs) { + messageHeader(1); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + schemaBlockHeader(table, tableId, version, loHiPairs.length / 2, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + for (long word : loHiPairs) { + Assert.assertEquals(word, i64()); + } + eof(); + } + + private void twoInferredAdoptionBlocks(String table) { + messageHeader(2); + Assert.assertEquals(0, varint()); + Assert.assertEquals(0, varint()); + + Assert.assertEquals(table, string()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(1, varint()); + Assert.assertEquals(1, varint()); + Assert.assertEquals("id", string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, u8()); + Assert.assertEquals(0, u8()); + Assert.assertEquals(11, i64()); + + schemaBlockHeader(table, 31, 41, 1, "id", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(0x3132333435363738L, i64()); + Assert.assertEquals(0x2122232425262728L, i64()); + eof(); + } + + private void uuidOnlyTable(String table, int tableId, long version, long lo, long hi) { + schemaTable(table, tableId, version, 1, "failed_b", QwpConstants.TYPE_UUID); + Assert.assertEquals(0, u8()); + Assert.assertEquals(lo, i64()); + Assert.assertEquals(hi, i64()); + eof(); + } + + private int u8() { + return in.get() & 0xff; + } + + private int varint() { + int result = 0; + int shift = 0; + int b; + do { + b = u8(); + result |= (b & 0x7f) << shift; + shift += 7; + } while ((b & 0x80) != 0); + return result; + } + } + + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledgeData; + private final List dataFrames = new ArrayList<>(); + private final CountDownLatch pong = new CountDownLatch(1); + private final AtomicInteger describeRequests = new AtomicInteger(); + private final int designatedIndex; + private volatile int result; + private volatile boolean respondToDescribe = true; + private final int tableId; + private volatile long version; + private long ackSequence; + // The snapshot the client last received for "events", through DESCRIBE or ACK + // feedback. With feedback enabled, a data frame acknowledged while that + // snapshot is stale carries the update, as the real server does. + private int deliveredResult = -1; + private long deliveredVersion = -1; + private volatile boolean isSchemaFeedbackEnabled; + + private SchemaHandler(int result, int tableId, long version) { + this(result, tableId, version, true); + } + + private SchemaHandler(int result, int tableId, long version, boolean acknowledgeData) { + this(result, tableId, version, acknowledgeData, 2); + } + + private SchemaHandler( + int result, + int tableId, + long version, + boolean acknowledgeData, + int designatedIndex + ) { + this.result = result; + this.tableId = tableId; + this.version = version; + this.acknowledgeData = acknowledgeData; + this.designatedIndex = designatedIndex; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + describeRequests.incrementAndGet(); + if (!respondToDescribe) { + return; + } + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); + int nameLength = request.getShort(QwpConstants.HEADER_SIZE + 1 + Long.BYTES) & 0xffff; + byte[] name = new byte[nameLength]; + request.position(QwpConstants.HEADER_SIZE + 1 + Long.BYTES + Short.BYTES); + request.get(name); + String tableName = new String(name, StandardCharsets.UTF_8); + int responseTableId = tableId + ("b".equals(tableName) ? 1 : 0); + if ("events".equals(tableName)) { + deliveredResult = result; + deliveredVersion = version; + } + send(client, schemaControlFrame(schemaPayload(requestId, responseTableId))); + return; + } + dataFrames.add(data); + notifyAll(); + if (!acknowledgeData) { + return; + } + send(client, ack()); + } + + private byte[] ack() { + long sequence = ackSequence++; + if (!isSchemaFeedbackEnabled || (result == deliveredResult && version == deliveredVersion)) { + return QwpWireTestUtils.buildAck(sequence); + } + if (result != QwpSchemaProtocol.RESULT_KNOWN && result != QwpSchemaProtocol.RESULT_MISSING) { + return ackInvalidateAll(sequence); + } + deliveredResult = result; + deliveredVersion = version; + return ackWithSchemaUpdate(sequence, "events", schemaPayload(0, tableId)); + } + + private static void send(TestWebSocketServer.ClientHandler client, byte[] frame) { + try { + client.sendBinary(frame); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onOpen(TestWebSocketServer.ClientHandler client) { + try { + client.sendPing(new byte[]{0x51, 0x57, 0x50}); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void onPong(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length == 3 && data[0] == 0x51 && data[1] == 0x57 && data[2] == 0x50) { + pong.countDown(); + } + } + + private boolean awaitPong() throws InterruptedException { + return pong.await(5, TimeUnit.SECONDS); + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private synchronized List awaitDataFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.size() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for " + count + " data frames; received " + dataFrames.size()); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return new ArrayList<>(dataFrames); + } + + private byte[] schemaPayload(long requestId, int responseTableId) { + byte[] id = "id".getBytes(StandardCharsets.UTF_8); + byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); + byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1; + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + payloadLength += 4 + 8 + 2 + 2 + + 2 + id.length + 4 + 2 + + 2 + failed.length + 4 + 2 + + 2 + ts.length + 4 + 2; + } + ByteBuffer out = ByteBuffer.allocate(payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + out.putInt(responseTableId).putLong(version).putShort((short) designatedIndex).putShort((short) 3) + .putShort((short) id.length).put(id).putInt(ColumnType.UUID).putShort((short) 0) + .putShort((short) failed.length).put(failed).putInt(ColumnType.STRING).putShort((short) 0) + .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); + } + return out.array(); + } + } + + private static final class LongTextSchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final List dataFrames = new ArrayList<>(); + private final AtomicInteger describeRequests = new AtomicInteger(); + private final int tableId; + private volatile int targetType; + private volatile long version; + private long ackSequence; + // The version the client last received, through DESCRIBE or ACK feedback. + private long deliveredVersion = -1; + + private LongTextSchemaHandler(int tableId, long version, int targetType) { + this.tableId = tableId; + this.version = version; + this.targetType = targetType; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + describeRequests.incrementAndGet(); + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = request.getLong(QwpConstants.HEADER_SIZE + 1); + deliveredVersion = version; + send(client, schemaControlFrame(schemaPayload(requestId))); + return; + } + dataFrames.add(data); + notifyAll(); + long sequence = ackSequence++; + if (version == deliveredVersion) { + send(client, QwpWireTestUtils.buildAck(sequence)); + } else { + deliveredVersion = version; + send(client, ackWithSchemaUpdate(sequence, "events", schemaPayload(0))); + } + } + + private static void send(TestWebSocketServer.ClientHandler client, byte[] frame) { + try { + client.sendBinary(frame); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for LONG-to-text data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private synchronized List awaitDataFrames(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.size() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for " + count + " data frames; received " + dataFrames.size()); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return new ArrayList<>(dataFrames); + } + + private byte[] schemaPayload(long requestId) { + byte[] value = "value".getBytes(StandardCharsets.UTF_8); + byte[] failed = "failed_b".getBytes(StandardCharsets.UTF_8); + byte[] ts = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + + 2 + value.length + 4 + 2 + + 2 + failed.length + 4 + 2 + + 2 + ts.length + 4 + 2; + ByteBuffer out = ByteBuffer.allocate(payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(tableId).putLong(version).putShort((short) 2).putShort((short) 3) + .putShort((short) value.length).put(value).putInt(targetType).putShort((short) 0) + .putShort((short) failed.length).put(failed).putInt(ColumnType.UUID).putShort((short) 0) + .putShort((short) ts.length).put(ts).putInt(ColumnType.TIMESTAMP_NANO).putShort((short) 0); + return out.array(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 44f711a3d..a4c6dd062 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -43,6 +43,153 @@ */ public class QwpWebSocketEncoderTest { + @Test + public void testHeaderRejectsTableCountOutsideUnsignedShortRange() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + encoder.writeHeader(0, 0); + encoder.writeHeader(0xffff, 0); + assertIllegalArgument(() -> encoder.writeHeader(-1, 0)); + assertIllegalArgument(() -> encoder.writeHeader(0x1_0000, 0)); + } + }); + } + + private static void assertIllegalArgument(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertIllegalState(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + // expected + } + } + + @Test + public void testEncodeSchemaKnownAndUnknownIdentityWire() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + buffer.getOrCreateColumn("x", TYPE_LONG, true).addLong(7); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer, 42, 19); + long p = encoder.getBuffer().getBufferPtr(); + Assert.assertEquals(FLAG_GORILLA | FLAG_SCHEMA, Unsafe.getUnsafe().getByte(p + HEADER_OFFSET_FLAGS)); + Assert.assertEquals(1, Unsafe.getUnsafe().getByte(p + 14)); + Assert.assertEquals(42, Unsafe.getUnsafe().getInt(p + 15)); + Assert.assertEquals(19, Unsafe.getUnsafe().getLong(p + 19)); + Assert.assertEquals(size - HEADER_SIZE, Unsafe.getUnsafe().getInt(p + 8)); + + size = encoder.encodeSchema(buffer, -1, -1); + p = encoder.getBuffer().getBufferPtr(); + Assert.assertEquals(0, Unsafe.getUnsafe().getByte(p + 14)); + Assert.assertEquals(1, Unsafe.getUnsafe().getByte(p + 15)); + Assert.assertEquals(size - HEADER_SIZE, Unsafe.getUnsafe().getInt(p + 8)); + } + }); + } + + @Test + public void testSchemaMessageRejectsMixedModesAndInvalidIdentity() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertIllegalArgument(() -> encoder.encodeSchema(buffer, -1, 1)); + assertIllegalArgument(() -> encoder.encodeSchema(buffer, 0, -1)); + assertIllegalArgument(() -> encoder.beginSchemaMessage(0, new GlobalSymbolDictionary(), -1, -1)); + encoder.beginSchemaMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertIllegalState(() -> encoder.addTable(buffer)); + encoder.beginMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertIllegalState(() -> encoder.addSchemaTable(buffer, 1, 1)); + } + }); + } + + @Test + public void testSchemaMultiTableAndLegacyReuseHaveExactFlags() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer first = new QwpTableBuffer("a"); + QwpTableBuffer second = new QwpTableBuffer("b")) { + first.getOrCreateColumn("x", TYPE_LONG, true).addLong(1); + first.nextRow(); + second.getOrCreateColumn("x", TYPE_LONG, true).addLong(2); + second.nextRow(); + int legacySize = encoder.encode(first); + byte[] expectedLegacy = new byte[legacySize]; + for (int i = 0; i < legacySize; i++) { + expectedLegacy[i] = Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + i); + } + encoder.beginSchemaMessage(2, new GlobalSymbolDictionary(), -1, -1); + encoder.addSchemaTable(first, 7, 8); + encoder.addSchemaTable(second, -1, -1); + int size = encoder.finishMessage(); + long p = encoder.getBuffer().getBufferPtr(); + Assert.assertEquals(FLAG_GORILLA | FLAG_DELTA_SYMBOL_DICT | FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(p + HEADER_OFFSET_FLAGS)); + Assert.assertEquals(2, Unsafe.getUnsafe().getShort(p + 6)); + Assert.assertEquals(size - HEADER_SIZE, Unsafe.getUnsafe().getInt(p + 8)); + + Assert.assertEquals(legacySize, encoder.encode(first)); + Assert.assertEquals(FLAG_GORILLA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + HEADER_OFFSET_FLAGS)); + for (int i = 0; i < legacySize; i++) { + Assert.assertEquals("legacy byte " + i, expectedLegacy[i], + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + i)); + } + } + }); + } + + @Test + public void testSchemaSplitPreservesIdentityBodyAndPerSplitDeferFlag() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + MicrobatchBuffer firstSplit = new MicrobatchBuffer(64); + MicrobatchBuffer secondSplit = new MicrobatchBuffer(64); + QwpTableBuffer first = new QwpTableBuffer("a"); + QwpTableBuffer second = new QwpTableBuffer("b")) { + first.getOrCreateColumn("x", TYPE_LONG, true).addLong(11); + first.nextRow(); + second.getOrCreateColumn("x", TYPE_LONG, true).addLong(22); + second.nextRow(); + encoder.beginSchemaMessage(2, new GlobalSymbolDictionary(), -1, -1); + int firstOffset = encoder.getBuffer().getPosition(); + encoder.addSchemaTable(first, 7, 8); + int firstLength = encoder.getBuffer().getPosition() - firstOffset; + int secondOffset = encoder.getBuffer().getPosition(); + encoder.addSchemaTable(second, 9, 10); + int secondLength = encoder.getBuffer().getPosition() - secondOffset; + encoder.finishMessage(); + + encoder.copySplitMessage(firstSplit, firstOffset, firstLength, true, -1, -1); + encoder.copySplitMessage(secondSplit, secondOffset, secondLength, false, -1, -1); + Assert.assertEquals(FLAG_GORILLA | FLAG_DELTA_SYMBOL_DICT | FLAG_SCHEMA | FLAG_DEFER_COMMIT, + Unsafe.getUnsafe().getByte(firstSplit.getBufferPtr() + HEADER_OFFSET_FLAGS)); + Assert.assertEquals(FLAG_GORILLA | FLAG_DELTA_SYMBOL_DICT | FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(secondSplit.getBufferPtr() + HEADER_OFFSET_FLAGS)); + assertSliceEquals(encoder.getBuffer().getBufferPtr() + firstOffset, firstSplit, firstLength); + assertSliceEquals(encoder.getBuffer().getBufferPtr() + secondOffset, secondSplit, secondLength); + } + }); + } + + private static void assertSliceEquals(long expected, MicrobatchBuffer split, int length) { + long actual = split.getBufferPtr() + HEADER_SIZE + 2; // empty delta start/count varints + for (int i = 0; i < length; i++) { + Assert.assertEquals("table body byte " + i, + Unsafe.getUnsafe().getByte(expected + i), Unsafe.getUnsafe().getByte(actual + i)); + } + } + @Test public void testBufferResetAndReuse() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java index 3911f5c05..f823bcc1b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderTest.java @@ -44,6 +44,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -185,7 +186,8 @@ public void testBinaryColumnAfterCloseThrows() throws Exception { @Test public void testBinaryColumnRejectsNullArray() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.binaryColumn("x", (byte[]) null); @@ -200,7 +202,8 @@ public void testBinaryColumnRejectsNullArray() throws Exception { @Test public void testBinaryColumnRejectsNullDirectByteSlice() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.binaryColumn("x", (io.questdb.client.std.bytes.DirectByteSlice) null); @@ -257,7 +260,8 @@ public void testCancelRowAfterCloseThrows() throws Exception { @Test public void testCancelRowDiscardsPartialRow() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("test"); sender.longColumn("x", 1); sender.boolColumn("y", true); @@ -282,6 +286,28 @@ public void testCancelRowNoOpWithoutTable() throws Exception { }); } + @Test + public void testSelectedTableSupportsEmptyFlushAndImplicitRows() throws Exception { + assertMemoryLeak(() -> { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { + sender.table("t"); + sender.flush(); + + sender.longColumn("v", 1).atNow(); + sender.longColumn("discarded", 2); + sender.cancelRow(); + sender.longColumn("v", 3).atNow(); + + sender.longColumn("v", 4); + sender.table("t"); // same-table selection does not disturb a partial row + sender.boolColumn("flag", true).atNow(); + + Assert.assertEquals(3, sender.getTableBuffer("t").getRowCount()); + } + }); + } + @Test public void testCloseIdemponent() throws Exception { assertMemoryLeak(() -> { @@ -416,7 +442,6 @@ public void testIpv4ColumnStringNullReturnsThis() throws Exception { // return the sender so chained builders keep working. assertMemoryLeak(() -> { try (QwpWebSocketSender sender = createUnconnectedSender()) { - sender.table("t"); Assert.assertSame(sender, sender.ipv4Column("addr", null)); } }); @@ -430,7 +455,8 @@ public void testIpv4ColumnStringRejectsNullLiteral() throws Exception { // real IPv4". Reject it as an invalid IPv4 address; the user can // pass a null reference (or omit the setter) to mark the row null. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); for (String input : new String[]{"null", "NULL", "Null", "nUlL"}) { try { @@ -456,7 +482,8 @@ public void testIpv4ColumnStringRejectsZeroAddress() throws Exception { // valid dotted-quad. Reject it on the client so the user gets a // clear error instead of a value that disappears on read. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = createUnconnectedSender()) { + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t"); try { sender.ipv4Column("addr", "0.0.0.0"); @@ -560,15 +587,8 @@ public void testOperationsAfterCloseThrow() throws Exception { @Test public void testPendingBytesMatchesGroundTruthAcrossTableSwitches() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - // Bypass ensureConnected: sendRow only short-circuits on - // connected==true and we don't drive any path that touches - // the cursor engine in this test. - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { // Round-robin three tables to exercise the per-row delta // logic across switches. The running pendingBytes must @@ -590,12 +610,8 @@ public void testPendingBytesMatchesGroundTruthAcrossTableSwitches() throws Excep @Test public void testPendingBytesUnchangedByCancelRow() throws Exception { assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.table("t0").longColumn("a", 1).longColumn("b", 2).atNow(); long committed = sender.getPendingBytes(); @@ -636,12 +652,8 @@ public void testRowAcceptedWhenServerAdvertisesNoCap() throws Exception { // the volatile serverMaxBatchSize to 0 mid-row, and a torn read that observed // the drop between the guard and the throw must not spuriously reject the row. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(0); Assert.assertEquals(0, sender.getServerMaxBatchSize()); @@ -665,12 +677,8 @@ public void testRowExceedingServerBatchCapThrows() throws Exception { // already exceed the server's advertised cap, before nextRow() commits it, so // the flush cannot build an oversize WS frame the server closes with 1009. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(64); Assert.assertEquals(64, sender.getServerMaxBatchSize()); @@ -706,12 +714,8 @@ public void testCumulativeBufferedBytesDoNotTripPerRowCap() throws Exception { // row once the table's CUMULATIVE buffered bytes crossed the cap; this // pins the wiring the single-row tests cannot distinguish. assertMemoryLeak(() -> { - try (QwpWebSocketSender sender = QwpWebSocketSender.createForTesting( - "localhost", 9000, - /*autoFlushRows*/ Integer.MAX_VALUE, - /*autoFlushBytes*/ 0, - /*autoFlushIntervalNanos*/ 0L)) { - sender.setConnectedForTest(true); + try (TestWebSocketServer server = startLegacyServer(); + QwpWebSocketSender sender = connectLegacySender(server, Integer.MAX_VALUE, 0, 0L)) { sender.applyServerBatchSizeLimit(64); // Three rows of 22-26 bytes each: every row fits the 64-byte cap, @@ -721,6 +725,9 @@ public void testCumulativeBufferedBytesDoNotTripPerRowCap() throws Exception { } QwpTableBuffer buf = sender.getTableBuffer("t"); Assert.assertEquals(3, buf.getRowCount()); + // The accumulated frame is intentionally larger than the cap; + // this test covers the per-row guard, not close-time flushing. + sender.reset(); } }); } @@ -857,12 +864,71 @@ public void testIllegalTableNameIsEscapedInTheMessage() throws Exception { }); } + @Test + public void testUnpairedSurrogateTableNameIsRejectedBeforeConnect() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketSender sender = createUnconnectedSender()) { + for (char invalidChar : new char[]{'\uD800', '\uDB80', '\uDFFF'}) { + try { + sender.table("bad" + invalidChar + "name"); + Assert.fail("an unpaired surrogate must be rejected"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage().contains("table name contains illegal characters")); + } + } + } + }); + } + private static MicrobatchBuffer getMicrobatchBuffer(QwpWebSocketSender sender, String fieldName) throws Exception { Field field = QwpWebSocketSender.class.getDeclaredField(fieldName); field.setAccessible(true); return (MicrobatchBuffer) field.get(sender); } + private static QwpWebSocketSender connectLegacySender( + TestWebSocketServer server, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos + ) { + CursorSendEngine engine = new CursorSendEngine(null, 4L * 1024 * 1024); + try { + return QwpWebSocketSender.connect( + "localhost", server.getPort(), null, + autoFlushRows, autoFlushBytes, autoFlushIntervalNanos, + null, false, engine, 0L); + } catch (Throwable t) { + engine.close(); + throw t; + } + } + + private static TestWebSocketServer startLegacyServer() throws Exception { + AtomicLong nextSequence = new AtomicLong(); + TestWebSocketServer server = new TestWebSocketServer(new TestWebSocketServer.WebSocketServerHandler() { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(QwpWireTestUtils.buildAck(nextSequence.getAndIncrement())); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + }); + boolean started = false; + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + started = true; + return server; + } finally { + if (!started) { + server.close(); + } + } + } + /** * Creates a sender without connecting. * For unit tests that don't need actual connectivity. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java new file mode 100644 index 000000000..ae10a518f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketSenderUnboundedWaitTest.java @@ -0,0 +1,146 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * {@code Long.MAX_VALUE} as a wait budget means "no limit". It once overflowed an + * absolute nanoTime deadline, so the waits below returned at once instead. + */ +public class QwpWebSocketSenderUnboundedWaitTest { + + @Test(timeout = 20_000) + public void testCloseFlushWaitsWithoutLimit() throws Exception { + assertMemoryLeak(() -> { + HeldAckHandler handler = new HeldAckHandler(); + try (TestWebSocketServer server = start(handler)) { + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;close_flush_timeout_millis=" + Long.MAX_VALUE + ";"); + sender.table("t").longColumn("v", 1).atNow(); + sender.flush(); + Assert.assertTrue(handler.received.await(5, TimeUnit.SECONDS)); + AtomicReference error = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + sender.close(); + } catch (Throwable t) { + error.set(t); + } + }, "unbounded-close"); + closer.start(); + try { + closer.join(200); + Assert.assertTrue("close() must wait for the ACK", closer.isAlive()); + } finally { + handler.release.countDown(); + closer.join(10_000); + } + Assert.assertFalse(closer.isAlive()); + Assert.assertNull(error.get()); + Assert.assertTrue(handler.acked.get()); + } + }); + } + + @Test(timeout = 20_000) + public void testDrainWaitsWithoutLimit() throws Exception { + assertMemoryLeak(() -> { + HeldAckHandler handler = new HeldAckHandler(); + try (TestWebSocketServer server = start(handler); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";schema_mode=off;close_flush_timeout_millis=0;")) { + sender.table("t").longColumn("v", 1).atNow(); + AtomicReference drained = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + Thread drainer = new Thread(() -> { + try { + drained.set(sender.drain(Long.MAX_VALUE)); + } catch (Throwable t) { + error.set(t); + } + }, "unbounded-drain"); + drainer.start(); + try { + Assert.assertTrue(handler.received.await(5, TimeUnit.SECONDS)); + drainer.join(200); + Assert.assertTrue("drain() must wait for the ACK", drainer.isAlive()); + } finally { + handler.release.countDown(); + drainer.join(10_000); + } + Assert.assertFalse(drainer.isAlive()); + Assert.assertNull(error.get()); + Assert.assertEquals(Boolean.TRUE, drained.get()); + } + }); + } + + private static TestWebSocketServer start(HeldAckHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + try { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } catch (Throwable t) { + server.close(); + throw t; + } + } + + // Holds the ACK for the first data frame until the test releases it. + private static final class HeldAckHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicBoolean acked = new AtomicBoolean(); + private final CountDownLatch received = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private long sequence; + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + received.countDown(); + try { + if (!release.await(10, TimeUnit.SECONDS)) { + return; + } + client.sendBinary(QwpWireTestUtils.buildAck(sequence++)); + acked.set(true); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java index b70bdc4f7..299bba248 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/WebSocketResponseTest.java @@ -400,7 +400,8 @@ public void testUnknownStatusByte() throws Exception { int size = WebSocketResponse.MIN_ERROR_RESPONSE_SIZE; long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); try { - Unsafe.getUnsafe().putByte(ptr, (byte) 0xFF); + // Unknown low-six-bit status remains a legacy error response. + Unsafe.getUnsafe().putByte(ptr, (byte) 0x3F); Unsafe.getUnsafe().putLong(ptr + 1, 1L); Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); @@ -409,8 +410,8 @@ public void testUnknownStatusByte() throws Exception { WebSocketResponse parsed = new WebSocketResponse(); Assert.assertTrue(parsed.readFrom(ptr, size)); - Assert.assertEquals((byte) 0xFF, parsed.getStatus()); - Assert.assertEquals("UNKNOWN(255)", parsed.getStatusName()); + Assert.assertEquals((byte) 0x3F, parsed.getStatus()); + Assert.assertEquals("UNKNOWN(63)", parsed.getStatusName()); } finally { Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index 998b6e585..046eba3b6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -25,9 +25,11 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; @@ -789,6 +791,29 @@ public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception }); } + @Test + public void testSchemaRequirementTracksUnackedSchemaFrames() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 1 << 20); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer rows = schemaTestRows()) { + assertEquals(0L, appendLegacy(engine, encoder, rows)); + assertFalse(engine.requiresSchema()); + assertEquals(1L, appendSchema(engine, encoder, rows)); + assertTrue(engine.requiresSchema()); + assertEquals(2L, appendLegacy(engine, encoder, rows)); + assertTrue(engine.acknowledge(0L)); + assertTrue("the schema frame is still unacked", engine.requiresSchema()); + assertTrue(engine.acknowledge(1L)); + assertFalse(engine.requiresSchema()); + assertEquals(3L, appendSchema(engine, encoder, rows)); + assertTrue(engine.requiresSchema()); + assertTrue(engine.acknowledge(3L)); + assertFalse(engine.requiresSchema()); + } + }); + } + @Test public void testSealedSegmentsAccumulateAfterRotation() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -1038,6 +1063,14 @@ private static Thread workerThread(SegmentManager manager) throws Exception { // a bounded wait. The probe is a second mapping of the same file the manager // worker writes through; its MAP_SHARED reads observe the worker's writes, and // it is closed before the next session opens the slot. + private static long appendLegacy(CursorSendEngine engine, QwpWebSocketEncoder encoder, QwpTableBuffer rows) { + return engine.appendBlocking(encoder.getBuffer().getBufferPtr(), encoder.encode(rows)); + } + + private static long appendSchema(CursorSendEngine engine, QwpWebSocketEncoder encoder, QwpTableBuffer rows) { + return engine.appendBlocking(encoder.getBuffer().getBufferPtr(), encoder.encodeSchema(rows, -1, -1)); + } + private static void awaitManagerPersistedWatermark(String slotDir, long expected) { long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); long last = AckWatermark.INVALID; @@ -1055,4 +1088,11 @@ private static void awaitManagerPersistedWatermark(String slotDir, long expected + " within 10s (last on-disk read=" + last + ")"); } + private static QwpTableBuffer schemaTestRows() { + QwpTableBuffer rows = new QwpTableBuffer("schema_requirement"); + rows.getOrCreateColumn("n", QwpConstants.TYPE_LONG, true).addLong(1); + rows.nextRow(); + return rows; + } + } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java new file mode 100644 index 000000000..be9e574b2 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaCoordinatorTest.java @@ -0,0 +1,520 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class CursorWebSocketSchemaCoordinatorTest { + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testColdHitCaseAliasAndForcedRefreshUseProductionTransport() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("cold").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + try { + loop.resolveSchema(new String(new char[QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH + 1]), 5_000); + Assert.fail("expected invalid table name"); + } catch (IllegalArgumentException expected) { + Assert.assertEquals(0, handler.requests.get()); + } + QwpSchemaResponse first = loop.resolveSchema("Trades", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, first.getResult()); + Assert.assertEquals(1, handler.requests.get()); + Assert.assertEquals(1, loop.resolveSchema("trades", 5_000).getTableId()); + Assert.assertEquals(1, handler.requests.get()); + Assert.assertEquals(2, loop.refreshSchema("TRADES", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testMissingIsCachedButRemoteFailuresAreTyped() throws Exception { + DescribeHandler missing = new DescribeHandler(true, QwpSchemaProtocol.RESULT_MISSING); + try (TestWebSocketServer server = server(missing); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("missing").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("Absent", 5_000).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("ABSENT", 0).getResult()); + Assert.assertEquals(1, missing.requests.get()); + } finally { + loop.close(); + } + } + assertRemoteReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "denied"); + assertRemoteReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "unavailable"); + } + + @Test + public void testTooLargeIsCachedButUnavailableIsNot() throws Exception { + // A table too wide to describe stays too wide until its version changes, + // which ACK feedback reports; re-asking every batch would cost a round trip. + DescribeHandler tooLarge = new DescribeHandler(true, QwpSchemaProtocol.RESULT_TOO_LARGE); + try (TestWebSocketServer server = server(tooLarge); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("too-large").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("Wide", 5_000).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("WIDE", 0).getResult()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.peekSchema("wide").getResult()); + Assert.assertEquals(1, tooLarge.requests.get()); + } finally { + loop.close(); + } + } + DescribeHandler unavailable = new DescribeHandler(true, QwpSchemaProtocol.RESULT_UNAVAILABLE); + try (TestWebSocketServer server = server(unavailable); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("unavailable").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.resolveSchema("t", 5_000)); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(2, unavailable.requests.get()); + Assert.assertNull(loop.peekSchema("t")); + } finally { + loop.close(); + } + } + } + + @Test + public void testTooLargeFeedbackIsCached() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("too-large-feedback").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + loop.resolveSchema("seed", 5_000); + handler.sendFeedback("Wide", QwpSchemaProtocol.RESULT_TOO_LARGE); + // The marker reply follows the feedback on the wire, so the I/O thread + // has applied the feedback once the marker lookup returns. + loop.refreshSchema("marker", 5_000); + int afterMarker = handler.requests.get(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_TOO_LARGE, loop.resolveSchema("wide", 0).getResult()); + Assert.assertEquals(afterMarker, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testOrderedFeedbackInvalidationAndDurableDemultiplexing() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler, true); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("feedback").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort(), true, true)) { + Assert.assertTrue(client.isServerDurableAckEnabled()); + CursorWebSocketSendLoop loop = loop(client, engine, true); + try { + loop.start(); + loop.resolveSchema("seed", 5_000); + + handler.sendFeedback("Feedback", QwpSchemaProtocol.RESULT_MISSING); + handler.durableBeforeReply = true; + loop.refreshSchema("marker", 5_000); + int afterMarker = handler.requests.get(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("feedback", 0).getResult()); + Assert.assertEquals(afterMarker, handler.requests.get()); + + handler.sendInvalidation(); + loop.refreshSchema("marker", 5_000); + loop.resolveSchema("feedback", 5_000); + Assert.assertEquals(afterMarker + 2, handler.requests.get()); + Assert.assertEquals(-1, engine.ackedFsn()); + Assert.assertEquals(-1, engine.publishedFsn()); + } finally { + loop.close(); + } + } + } + + @Test + public void testFailedRefreshEvictsPriorCachedSchema() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("refresh-eviction").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertEquals(1, loop.resolveSchema("cached", 5_000).getTableId()); + handler.reply = false; + Future refresh = executor.submit(() -> loop.refreshSchema("CACHED", 200)); + Assert.assertTrue(handler.awaitRequests(2)); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, refresh); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("cached", 0)); + handler.reply = true; + Assert.assertEquals(3, loop.resolveSchema("cached", 5_000).getTableId()); + Assert.assertEquals(3, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testTimeoutIgnoresLateReplyAndNextRequestSucceeds() throws Exception { + DescribeHandler handler = new DescribeHandler(false); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("late").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Future timedOut = executor.submit(() -> loop.resolveSchema("late", 200)); + Assert.assertTrue(handler.awaitFirstRequest()); + assertFutureReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, timedOut); + handler.replyToFirst(); + handler.reply = true; + Assert.assertEquals(2, loop.resolveSchema("late", 5_000).getTableId()); + Assert.assertEquals(2, handler.requests.get()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testAbandonOnDisconnectFailsPendingLookupBeforeItsDeadline() throws Exception { + DescribeHandler handler = new DescribeHandler(false); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("abandon").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + loop.setAbandonSchemaLookupOnDisconnect(true); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + loop.start(); + Assert.assertTrue(loop.isWireUp()); + // A budget far beyond the assertion window: only the drop can end it. + Future pending = executor.submit(() -> loop.resolveSchema("t", 60_000)); + Assert.assertTrue(handler.awaitFirstRequest()); + server.close(); + try { + pending.get(5, TimeUnit.SECONDS); + Assert.fail("lookup survived the connection loss"); + } catch (ExecutionException e) { + LineSenderSchemaException cause = (LineSenderSchemaException) e.getCause(); + Assert.assertEquals(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, cause.getReason()); + Assert.assertTrue(cause.getMessage(), cause.getMessage().contains("connection lost")); + } + Assert.assertFalse(loop.isWireUp()); + } finally { + executor.shutdownNow(); + loop.close(); + } + } + } + + @Test + public void testPeekReturnsOnlyCachedSchema() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("peek").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + Assert.assertNull(loop.peekSchema("Trades")); + Assert.assertEquals(0, handler.requests.get()); + QwpSchemaResponse resolved = loop.resolveSchema("Trades", 5_000); + Assert.assertSame(resolved, loop.peekSchema("trades")); + Assert.assertEquals(1, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + @Test + public void testUnnegotiatedConnectionIsUnavailable() throws Exception { + io.questdb.client.test.tools.TestUtils.assertMemoryLeak(() -> { + DescribeHandler handler = new DescribeHandler(true); + try (TestWebSocketServer server = new TestWebSocketServer(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("legacy").getAbsolutePath(), 1 << 20)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (WebSocketClient client = connect(server.getPort(), false)) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(0, handler.requests.get()); + } finally { + loop.close(); + } + } + } + }); + } + + @Test + public void testMalformedControlFailsLookupWithoutAckFallback() throws Exception { + DescribeHandler handler = new DescribeHandler(true); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder("malformed-control").getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + () -> loop.resolveSchema("t", 5_000)); + Assert.assertEquals(1, handler.requests.get()); + } finally { + loop.close(); + } + } + } + + private static CursorWebSocketSendLoop loop(WebSocketClient client, CursorSendEngine engine) { + return loop(client, engine, false); + } + + private static CursorWebSocketSendLoop loop(WebSocketClient client, CursorSendEngine engine, boolean durable) { + return new CursorWebSocketSendLoop(client, engine, 0, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, null, 1, 4, durable); + } + + private void assertRemoteReason(int result, LineSenderSchemaException.Reason reason, String directory) throws Exception { + DescribeHandler handler = new DescribeHandler(true, result); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = new CursorSendEngine(temp.newFolder(directory).getAbsolutePath(), 1 << 20); + WebSocketClient client = connect(server.getPort())) { + CursorWebSocketSendLoop loop = loop(client, engine); + try { + loop.start(); + assertReason(reason, () -> loop.resolveSchema("t", 5_000)); + } finally { + loop.close(); + } + } + } + + private static TestWebSocketServer server(DescribeHandler handler) throws Exception { + return server(handler, false); + } + + private static TestWebSocketServer server(DescribeHandler handler, boolean durable) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler, durable); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static WebSocketClient connect(int port) { + return connect(port, true); + } + + private static WebSocketClient connect(int port, boolean schema) { + return connect(port, schema, false); + } + + private static WebSocketClient connect(int port, boolean schema, boolean durable) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + client.connect("localhost", port); + if (schema) { + client.requestQwpSchema(); + } + client.setQwpRequestDurableAck(durable); + client.upgrade("/write/v4", 5_000, null); + return client; + } + + private static void assertReason(LineSenderSchemaException.Reason reason, Runnable action) { + try { + action.run(); + Assert.fail("expected schema failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + } + } + + private static void assertFutureReason( + LineSenderSchemaException.Reason reason, + Future future + ) throws Exception { + try { + future.get(5, TimeUnit.SECONDS); + Assert.fail("expected schema failure"); + } catch (ExecutionException e) { + Assert.assertTrue(e.getCause() instanceof LineSenderSchemaException); + Assert.assertEquals(reason, ((LineSenderSchemaException) e.getCause()).getReason()); + } + } + + private static File taskTempRoot() { + File root = new File("target/schema-coordination-2.7/tmp").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class DescribeHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger requests = new AtomicInteger(); + private final CountDownLatch firstRequest = new CountDownLatch(1); + private volatile boolean reply; + private volatile boolean durableBeforeReply; + private volatile boolean malformedReply; + private TestWebSocketServer.ClientHandler client; + private long firstRequestId; + private final int result; + + private DescribeHandler(boolean reply) { + this(reply, QwpSchemaProtocol.RESULT_KNOWN); + } + + private DescribeHandler(boolean reply, int result) { + this.reply = reply; + this.result = result; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, + ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).getInt()); + Assert.assertEquals(QwpSchemaProtocol.FLAG_CONTROL, data[5]); + long requestId = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN).getLong(QwpConstants.HEADER_SIZE + 1); + this.client = client; + int requestCount = requests.incrementAndGet(); + notifyAll(); + if (requestCount == 1) { + firstRequestId = requestId; + firstRequest.countDown(); + } + if (reply) { + if (malformedReply) { + ByteBuffer malformed = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN); + sendRaw(client, malformed.putInt(QwpConstants.MAGIC_MESSAGE).array()); + return; + } + if (durableBeforeReply) { + durableBeforeReply = false; + sendRaw(client, new byte[]{WebSocketResponse.STATUS_DURABLE_ACK, 0, 0}); + } + send(client, requestId, requests.get(), result); + } + } + + synchronized void replyToFirst() throws IOException { + send(client, firstRequestId, 1, result); + } + + boolean awaitFirstRequest() throws InterruptedException { + return firstRequest.await(5, TimeUnit.SECONDS); + } + + synchronized boolean awaitRequests(int count) throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (requests.get() < count) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + return false; + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return true; + } + + synchronized void sendFeedback(String tableName, int result) { + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + byte[] schema = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0).put((byte) result).array(); + ByteBuffer b = ByteBuffer.allocate(11 + 2 + 2 + name.length + 4 + schema.length) + .order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(0).putShort((short) 0).putShort((short) 1) + .putShort((short) name.length).put(name).putInt(schema.length).put(schema); + sendRaw(client, b.array()); + } + + synchronized void sendInvalidation() { + ByteBuffer b = ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN); + b.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_INVALIDATE_ALL)) + .putLong(0).putShort((short) 0); + sendRaw(client, b.array()); + } + + private static void send(TestWebSocketServer.ClientHandler client, long requestId, int tableId, int result) { + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1; + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + payloadLength += 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + } + ByteBuffer b = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + b.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payloadLength).put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId).put((byte) result); + if (result == QwpSchemaProtocol.RESULT_KNOWN) { + b.putInt(tableId).putLong(1).putShort((short) -1).putShort((short) 1) + .putShort((short) column.length).put(column).putInt(ColumnType.LONG).putShort((short) 0); + } + sendRaw(client, b.array()); + } + + private static void sendRaw(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + client.sendBinary(data); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java new file mode 100644 index 000000000..b47f3da82 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSchemaLifecycleTest.java @@ -0,0 +1,747 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.rules.TestRule; +import org.junit.runners.model.Statement; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +public class CursorWebSocketSchemaLifecycleTest { + @Rule + public final TestRule memoryLeak = (base, description) -> new Statement() { + @Override + public void evaluate() throws Throwable { + AtomicReference failure = new AtomicReference<>(); + TestUtils.assertMemoryLeak(() -> { + try { + base.evaluate(); + } catch (Throwable e) { + failure.set(e); + } + }); + if (failure.get() != null) { + throw failure.get(); + } + } + }; + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testCloseReleasesLongDeadlineWaiterBeforeLoopStarts() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("close-before-start"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + Lookup lookup = lookup(loop, "never-started", Long.MAX_VALUE, null); + Assert.assertTrue(lookup.entered.await(5, TimeUnit.SECONDS)); + loop.close(); + lookup.join(); + assertUnavailable(lookup.error.get()); + } + } + + @Test + public void testCloseReleasesLookupAlreadySentByActiveLoop() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("close-active"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup lookup = lookup(loop, "active", Long.MAX_VALUE, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + loop.close(); + lookup.join(); + assertUnavailable(lookup.error.get()); + } + } + + @Test + public void testInterruptionRestoresFlagAndReleasesSlotForNextLookup() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("interrupt"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup interrupted = lookup(loop, "first", Long.MAX_VALUE, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + interrupted.thread.interrupt(); + interrupted.join(); + assertUnavailable(interrupted.error.get()); + Assert.assertTrue(interrupted.interrupted.get()); + + handler.reply = true; + QwpSchemaResponse next = loop.resolveSchema("second", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, next.getResult()); + Assert.assertEquals("second", handler.lastName.get()); + } + } + + @Test + public void testDisconnectRetriesCurrentLookupRetainsCacheAndRejectsStaleReply() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + QwpSchemaResponse cached = loop.resolveSchema("cached", 5_000); + handler.disconnectName = "lost"; + handler.disconnectsRemaining = 2; + Lookup lost = lookup(loop, "lost", 5_000, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + Assert.assertSame(cached, loop.resolveSchema("CACHED", 5_000)); + Lookup busy = lookup(loop, "fresh", 5_000, null); + busy.join(); + assertUnavailable(busy.error.get()); + Assert.assertTrue(busy.error.get().getMessage().contains("another schema lookup")); + handler.sendStaleBeforeNextReply = true; + reconnect.allow.countDown(); + lost.join(); + Assert.assertNull(String.valueOf(lost.error.get()), lost.error.get()); + Assert.assertNotNull(lost.response.get()); + Assert.assertEquals(4, lost.response.get().getTableId()); + Assert.assertSame(lost.response.get(), loop.resolveSchema("lost", 0)); + Assert.assertEquals(3, server.handshakeCount()); + + // The cache survives the reconnect. ACK feedback on the new + // connection refreshes any entry that went stale meanwhile. + int requestsBeforeCacheProbe = handler.requests; + Assert.assertSame(cached, loop.resolveSchema("cached", 5_000)); + Assert.assertEquals(requestsBeforeCacheProbe, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testQueuedUnsentLookupSurvivesDisconnect() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("queued-disconnect"); + DisconnectingClient client = new DisconnectingClient()) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + reconnect.allow.countDown(); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Assert.assertTrue(client.receiving.await(5, TimeUnit.SECONDS)); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + client.disconnect.countDown(); + queued.join(); + Assert.assertNull(String.valueOf(queued.error.get()), queued.error.get()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, queued.response.get().getResult()); + Assert.assertEquals(1, handler.requests); + } finally { + client.disconnect.countDown(); + } + } + } + } + + @Test + public void testQueuedLookupBecomesUnavailableAtLegacyReplacement() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("queued-legacy-replacement"); + DisconnectingClient client = new DisconnectingClient()) { + server.setAdvertiseSchema(false); + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + reconnect.allow.countDown(); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Assert.assertTrue(client.receiving.await(5, TimeUnit.SECONDS)); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + client.disconnect.countDown(); + queued.join(); + assertUnavailable(queued.error.get()); + Assert.assertTrue(loop.isWireUp()); + Assert.assertFalse(loop.isSchemaEnabled()); + Assert.assertFalse(engine.requiresSchema()); + Assert.assertEquals(0, handler.requests); + } finally { + client.disconnect.countDown(); + } + } + } + } + + @Test + public void testReconnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + handler.disconnectName = "deadline"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-deadline"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup lookup = lookup(loop, "deadline", 1_000, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + Assert.assertTrue(lookup.error.get().getMessage().contains("timed out")); + reconnect.allow.countDown(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, + loop.resolveSchema("probe", 5_000).getResult()); + Assert.assertEquals(2, handler.requests); + Assert.assertEquals("probe", handler.lastName.get()); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testInterruptDuringReconnectReleasesSlotForQueuedLookup() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + handler.disconnectName = "interrupted"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-interrupt"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup interrupted = lookup(loop, "interrupted", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + interrupted.thread.interrupt(); + interrupted.join(); + assertUnavailable(interrupted.error.get()); + Assert.assertTrue(interrupted.interrupted.get()); + Lookup queued = lookup(loop, "queued", 5_000, null); + queued.awaitWaiting(); + reconnect.allow.countDown(); + queued.join(); + Assert.assertNull(String.valueOf(queued.error.get()), queued.error.get()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, queued.response.get().getResult()); + Assert.assertEquals(2, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testCloseDuringReconnectReleasesLookupBeforeFactoryReturns() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.disconnectName = "closed"; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-close"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + AtomicReference closeError = new AtomicReference<>(); + Thread closer = new Thread(() -> { + try { + loop.close(); + } catch (Throwable e) { + closeError.set(e); + } + }); + try { + loop.start(); + Lookup lookup = lookup(loop, "closed", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + closer.start(); + lookup.join(); + assertUnavailable(lookup.error.get()); + Assert.assertTrue(lookup.error.get().getMessage().contains("closed")); + } finally { + reconnect.allow.countDown(); + closer.join(5_000); + Assert.assertFalse(closer.isAlive()); + } + Assert.assertNull(String.valueOf(closeError.get()), closeError.get()); + Assert.assertEquals(1, handler.requests); + } + } + } + + @Test + public void testLookupsDuringOutageDoNotShortenReconnectBackoff() throws Exception { + SchemaHandler handler = new SchemaHandler(); + FailingReconnectFactory reconnect = new FailingReconnectFactory(); + try (CursorSendEngine engine = engine("outage-backoff")) { + TestWebSocketServer server = server(handler); + CursorWebSocketSendLoop loop = null; + try { + loop = new CursorWebSocketSendLoop(connect(server.getPort()), engine, 0, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, reconnect, 200, 200); + loop.start(); + server.close(); + Assert.assertTrue(reconnect.firstAttempt.await(5, TimeUnit.SECONDS)); + int attemptsBefore = reconnect.attempts.get(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(1); + int lookups = 0; + while (System.nanoTime() < deadline) { + try { + loop.resolveSchema("outage", 1); + Assert.fail("the wire is down"); + } catch (LineSenderSchemaException expected) { + lookups++; + } + } + int attempts = reconnect.attempts.get() - attemptsBefore; + // A 200 ms backoff plus up to 200 ms jitter allows at most 5 attempts per second. + Assert.assertTrue("lookups=" + lookups + ", attempts=" + attempts, attempts <= 6); + Assert.assertTrue("lookups=" + lookups, lookups > 50); + } finally { + if (loop != null) { + loop.close(); + } + server.close(); + } + } + } + + @Test + public void testMalformedCurrentReplyCancelsLookupEvenWithReconnectAvailable() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("reconnect-malformed"); + WebSocketClient client = connect(server.getPort())) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(client, engine, reconnect)) { + try { + loop.start(); + Lookup lookup = lookup(loop, "malformed", Long.MAX_VALUE, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + handler.malformedReply = false; + handler.reply = true; + reconnect.allow.countDown(); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, + loop.resolveSchema("probe", 5_000).getResult()); + Assert.assertEquals(2, handler.requests); + } finally { + reconnect.allow.countDown(); + } + } + } + } + + @Test + public void testAsyncInitialConnectWaitCountsAgainstOriginalLookupDeadline() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.reply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("async-initial")) { + GatedReconnectFactory reconnect = new GatedReconnectFactory(server.getPort()); + try (CursorWebSocketSendLoop loop = loop(null, engine, reconnect)) { + loop.start(); + Lookup lookup = lookup(loop, "deadline", 25, null); + Assert.assertTrue(reconnect.attempted.await(5, TimeUnit.SECONDS)); + lookup.join(); + assertUnavailable(lookup.error.get()); + reconnect.allow.countDown(); + Assert.assertTrue(reconnect.connected.await(5, TimeUnit.SECONDS)); + QwpSchemaResponse probe = loop.resolveSchema("probe", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, probe.getResult()); + Assert.assertEquals(1, handler.requests); + Assert.assertEquals("probe", handler.lastName.get()); + } finally { + reconnect.allow.countDown(); + } + } + } + + @Test + public void testMalformedTimedOutReplyDoesNotPoisonIdleConnection() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("stale-idle"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup stale = lookup(loop, "stale", 1_000, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + stale.join(); + assertUnavailable(stale.error.get()); + handler.sendMalformedLastRequestAndFeedback("idle_barrier"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + Assert.assertEquals(QwpSchemaProtocol.RESULT_MISSING, + loop.resolveSchema("idle_barrier", 0).getResult()); + break; + } catch (LineSenderSchemaException e) { + assertUnavailable(e); + if (System.nanoTime() >= deadline) { + Assert.fail("schema feedback barrier was not observed"); + } + Thread.yield(); + } + } + Assert.assertEquals(1, handler.requests); + handler.reply = true; + QwpSchemaResponse response = loop.resolveSchema("usable", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, response.getResult()); + Assert.assertEquals("usable", handler.lastName.get()); + Assert.assertEquals(1, server.handshakeCount()); + } + } + + @Test + public void testMalformedOldReplyDoesNotFailNewLookupOrPopulateOldCache() throws Exception { + SchemaHandler handler = new SchemaHandler(); + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("stale-pending"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup stale = lookup(loop, "stale", 1_000, null); + Assert.assertTrue(handler.requestSeen.await(5, TimeUnit.SECONDS)); + stale.join(); + assertUnavailable(stale.error.get()); + handler.sendStaleBeforeNextReply = true; + QwpSchemaResponse current = loop.resolveSchema("current", 5_000); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, current.getResult()); + Assert.assertEquals(2, current.getTableId()); + Assert.assertEquals(1, server.handshakeCount()); + + handler.reply = true; + QwpSchemaResponse staleProbe = loop.resolveSchema("stale", 5_000); + Assert.assertEquals(3, staleProbe.getTableId()); + Assert.assertEquals(3, handler.requests); + } + } + + @Test + public void testMalformedMatchingReplyFailsTransportAndReleasesWaiter() throws Exception { + SchemaHandler handler = new SchemaHandler(); + handler.malformedReply = true; + try (TestWebSocketServer server = server(handler); + CursorSendEngine engine = engine("malformed-current"); + WebSocketClient client = connect(server.getPort()); + CursorWebSocketSendLoop loop = loop(client, engine, null)) { + loop.start(); + Lookup lookup = lookup(loop, "current", Long.MAX_VALUE, null); + lookup.join(); + assertUnavailable(lookup.error.get()); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try { + loop.checkError(); + } catch (LineSenderException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Invalid schema control response:")); + break; + } + if (System.nanoTime() >= deadline) { + Assert.fail("malformed schema response did not publish the transport error"); + } + Thread.yield(); + } + } + } + + private CursorSendEngine engine(String name) throws IOException { + return new CursorSendEngine(temp.newFolder(name).getAbsolutePath(), 1 << 20); + } + + private static CursorWebSocketSendLoop loop( + WebSocketClient client, + CursorSendEngine engine, + CursorWebSocketSendLoop.ReconnectFactory reconnect + ) { + return new CursorWebSocketSendLoop( + client, engine, 0, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, reconnect, 1, 4); + } + + private static Lookup lookup( + CursorWebSocketSendLoop loop, + String table, + long timeoutMillis, + CountDownLatch completed + ) { + Lookup lookup = new Lookup(); + lookup.thread = new Thread(() -> { + lookup.entered.countDown(); + try { + lookup.response.set(loop.resolveSchema(table, timeoutMillis)); + } catch (Throwable e) { + lookup.error.set(e); + } finally { + lookup.interrupted.set(Thread.currentThread().isInterrupted()); + if (completed != null) { + completed.countDown(); + } + } + }, "schema-lookup-" + table); + lookup.thread.start(); + return lookup; + } + + private static void assertUnavailable(Throwable error) { + Assert.assertTrue(String.valueOf(error), error instanceof LineSenderSchemaException); + Assert.assertEquals( + LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, + ((LineSenderSchemaException) error).getReason()); + } + + private static TestWebSocketServer server(SchemaHandler handler) throws Exception { + TestWebSocketServer server = new TestWebSocketServer(handler); + server.setAdvertiseSchema(true); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + return server; + } + + private static WebSocketClient connect(int port) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + client.requestQwpSchema(); + client.upgrade("/write/v4", 5_000, null); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + private static File taskTempRoot() { + File root = new File("target/schema-coordination-2.7/tmp").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class Lookup { + private final AtomicReference error = new AtomicReference<>(); + private final CountDownLatch entered = new CountDownLatch(1); + private final AtomicBoolean interrupted = new AtomicBoolean(); + private final AtomicReference response = new AtomicReference<>(); + private Thread thread; + + private void awaitWaiting() throws InterruptedException { + Assert.assertTrue(entered.await(5, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (thread.getState() != Thread.State.TIMED_WAITING) { + Assert.assertTrue("lookup did not wait", thread.isAlive() && System.nanoTime() < deadline); + Thread.yield(); + } + } + + private void join() throws InterruptedException { + thread.join(5_000); + Assert.assertFalse("lookup thread did not finish", thread.isAlive()); + } + } + + private static final class FailingReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final CountDownLatch firstAttempt = new CountDownLatch(1); + + @Override + public WebSocketClient reconnect() { + attempts.incrementAndGet(); + firstAttempt.countDown(); + throw new RuntimeException("server is down"); + } + } + + private static final class GatedReconnectFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final CountDownLatch allow = new CountDownLatch(1); + private final CountDownLatch attempted = new CountDownLatch(1); + private final CountDownLatch connected = new CountDownLatch(1); + private final int port; + + private GatedReconnectFactory(int port) { + this.port = port; + } + + @Override + public WebSocketClient reconnect() { + attempted.countDown(); + try { + if (!allow.await(5, TimeUnit.SECONDS)) { + throw new AssertionError("replacement was not released"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + WebSocketClient client = connect(port); + connected.countDown(); + return client; + } + + @Override + public void setSchemaRequired(boolean isRequired) { + } + } + + // Fault injection only: hold the receive poll until the producer has queued + // a lookup, then fail before the I/O thread can send that lookup. + private static final class DisconnectingClient extends WebSocketClient { + private final CountDownLatch disconnect = new CountDownLatch(1); + private final CountDownLatch receiving = new CountDownLatch(1); + + private DisconnectingClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public boolean tryReceiveFrame(WebSocketFrameHandler handler) { + receiving.countDown(); + try { + Assert.assertTrue(disconnect.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + throw new LineSenderException("injected transport disconnect"); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + + private static final class SchemaHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicReference lastName = new AtomicReference<>(); + private final CountDownLatch requestSeen = new CountDownLatch(1); + private volatile String disconnectName; + private int disconnectsRemaining = 1; + private volatile TestWebSocketServer.ClientHandler lastClient; + private volatile long lastRequestId; + private volatile boolean malformedReply; + private volatile boolean reply; + private volatile boolean sendStaleBeforeNextReply; + private volatile int requests; + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + ByteBuffer input = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + long requestId = input.getLong(QwpConstants.HEADER_SIZE + 1); + int nameLength = input.getShort(QwpConstants.HEADER_SIZE + 9) & 0xffff; + String name = new String(data, QwpConstants.HEADER_SIZE + 11, nameLength, StandardCharsets.UTF_8); + requests++; + lastClient = client; + long staleRequestId = lastRequestId; + lastRequestId = requestId; + lastName.set(name); + requestSeen.countDown(); + if (name.equals(disconnectName) && disconnectsRemaining-- > 0) { + try { + client.sendClose(1001, "test disconnect"); + } catch (IOException e) { + throw new AssertionError(e); + } + } else if (sendStaleBeforeNextReply) { + sendStaleBeforeNextReply = false; + sendMalformed(client, staleRequestId); + send(client, staleRequestId, 999); + send(client, requestId, requests); + } else if (malformedReply) { + sendMalformed(client, requestId); + } else if (reply) { + send(client, requestId, requests); + } + } + + private synchronized void sendMalformedLastRequestAndFeedback(String tableName) { + sendMalformed(lastClient, lastRequestId); + byte[] name = tableName.getBytes(StandardCharsets.UTF_8); + byte[] schema = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(0) + .put((byte) QwpSchemaProtocol.RESULT_MISSING).array(); + ByteBuffer feedback = ByteBuffer.allocate(11 + 2 + 2 + name.length + 4 + schema.length) + .order(ByteOrder.LITTLE_ENDIAN); + feedback.put((byte) (WebSocketResponse.STATUS_OK | WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES)) + .putLong(0).putShort((short) 0).putShort((short) 1) + .putShort((short) name.length).put(name).putInt(schema.length).put(schema); + try { + lastClient.sendBinary(feedback.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static void sendMalformed(TestWebSocketServer.ClientHandler client, long requestId) { + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + 9).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(9).put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static void send(TestWebSocketServer.ClientHandler client, long requestId, int tableId) { + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + 8 + 1 + 4 + 8 + 2 + 2 + 2 + column.length + 4 + 2; + ByteBuffer out = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(QwpConstants.MAGIC_MESSAGE).put((byte) 1).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payloadLength).put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId).put((byte) QwpSchemaProtocol.RESULT_KNOWN).putInt(tableId) + .putLong(1).putShort((short) -1).putShort((short) 1) + .putShort((short) column.length).put(column).putInt(ColumnType.LONG).putShort((short) 0); + try { + client.sendBinary(out.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java deleted file mode 100644 index f2452dde0..000000000 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopErrorLatchTest.java +++ /dev/null @@ -1,251 +0,0 @@ -/*+***************************************************************************** - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2026 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -package io.questdb.client.test.cutlass.qwp.client.sf.cursor; - -import io.questdb.client.LineSenderServerException; -import io.questdb.client.SenderError; -import io.questdb.client.cutlass.line.LineSenderException; -import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.std.Unsafe; -import org.junit.Assert; -import org.junit.Test; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; - -/** - * Pinpointed tests for the latched-error contract on {@link CursorWebSocketSendLoop}: - * {@code recordFatal} → {@link CursorWebSocketSendLoop#getTerminalError} + - * {@link CursorWebSocketSendLoop#getLastTerminalServerError} + - * {@link CursorWebSocketSendLoop#checkError}. Bypasses the constructor entirely - * via {@code Unsafe.allocateInstance} to avoid the live wire/engine dependencies - * — the latch is a self-contained piece of state. - */ -public class CursorWebSocketSendLoopErrorLatchTest { - - @Test - public void testCheckErrorRethrowsLineSenderException() throws Exception { - // checkError must rethrow the SAME LineSenderException instance, not - // a wrapper. Producers depend on this so getServerError() works on - // typed throws. - CursorWebSocketSendLoop loop = newBareLoop(); - SenderError err = newSenderError(); - LineSenderServerException original = new LineSenderServerException(err); - setField(loop, "terminalError", original); - Assert.assertNull(loop.getSynchronouslySurfacedError()); - - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrown) { - Assert.assertSame("checkError must rethrow LineSenderException unchanged", - original, thrown); - Assert.assertSame(err, - ((LineSenderServerException) thrown).getServerError()); - } - Assert.assertSame("checkError must mark the exact latched throwable as user-owned", - original, loop.getSynchronouslySurfacedError()); - // a synchronously surfaced latch is owned -- the close() safety net - // must stay silent now - loop.checkUnsurfacedError(); - } - - @Test - public void testRecordFatalWrapsNonLineSenderThrowableOnce() throws Exception { - // Non-LineSenderException causes (NPE, IOException, etc.) are wrapped - // in a LineSenderException ONCE, at latch time -- so producers always - // see one exception type AND every rethrow delivers the same instance, - // which close() relies on to suppress double-signals by identity. - CursorWebSocketSendLoop loop = newBareLoop(); - Throwable raw = new RuntimeException("oh no"); - invokeRecordFatal(loop, raw); - Assert.assertNull(loop.getSynchronouslySurfacedError()); - - LineSenderException first = null; - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrown) { - Assert.assertNotSame(raw, thrown); - Assert.assertSame(raw, thrown.getCause()); - Assert.assertTrue(thrown.getMessage().contains("oh no")); - first = thrown; - } - Assert.assertSame("the latch must hold the wrapper, not the raw cause", - first, loop.getTerminalError()); - Assert.assertSame("ownership tracks the latched wrapper", - first, loop.getSynchronouslySurfacedError()); - loop.checkUnsurfacedError(); // owned -> silent - - try { - loop.checkError(); - Assert.fail("expected throw"); - } catch (LineSenderException thrownAgain) { - Assert.assertSame("every rethrow must deliver the same instance", - first, thrownAgain); - } - } - - @Test - public void testCheckErrorIsNoopWhenNoLatch() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Assert.assertNull(loop.getTerminalError()); - loop.checkError(); // must not throw - } - - @Test - public void testCheckUnsurfacedErrorThrowsOnceThenStaysSilent() throws Exception { - // The close() safety net: an unowned latch must rethrow exactly like - // checkError; once any synchronous caller has owned the error, it - // must stay silent -- unlike checkError, which keeps throwing. - CursorWebSocketSendLoop loop = newBareLoop(); - loop.checkUnsurfacedError(); // no latch -> silent - - LineSenderException e = new LineSenderException("wire fail"); - setField(loop, "terminalError", e); - try { - loop.checkUnsurfacedError(); - Assert.fail("an unowned latch must rethrow from the safety net"); - } catch (LineSenderException thrown) { - Assert.assertSame(e, thrown); - } - - loop.checkUnsurfacedError(); // the throw above made the caller owner -> silent - - try { - loop.checkError(); - Assert.fail("checkError must keep rethrowing a latched error"); - } catch (LineSenderException thrown) { - Assert.assertSame(e, thrown); - } - } - - @Test - public void testGetTerminalErrorReturnsLatchedThrowable() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Throwable e = new LineSenderException("boom"); - setField(loop, "terminalError", e); - Assert.assertSame(e, loop.getTerminalError()); - } - - @Test - public void testGetTerminalErrorIsNullBeforeAnyFailure() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - Assert.assertNull("loops with no latched error must report null", - loop.getTerminalError()); - } - - @Test - public void testRecordFatalLatchesThrowableOnly() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - // running must be true initially so we can verify recordFatal flips it. - setField(loop, "running", true); - Throwable e = new LineSenderException("wire fail"); - - invokeRecordFatal(loop, e); - - Assert.assertSame(e, loop.getTerminalError()); - Assert.assertNull("typed payload must be null for a wire-level fatal", - loop.getLastTerminalServerError()); - Assert.assertFalse("recordFatal must stop the loop", - (Boolean) getField(loop, "running")); - } - - @Test - public void testRecordFatalLatchesBothThrowableAndSenderError() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - setField(loop, "running", true); - SenderError err = newSenderError(); - LineSenderServerException ex = new LineSenderServerException(err); - - invokeRecordFatal(loop, ex); - - Assert.assertSame(ex, loop.getTerminalError()); - Assert.assertSame("typed payload is derived from the latched LineSenderServerException", - err, loop.getLastTerminalServerError()); - Assert.assertFalse((Boolean) getField(loop, "running")); - } - - @Test - public void testRecordFatalIsIdempotent() throws Exception { - CursorWebSocketSendLoop loop = newBareLoop(); - setField(loop, "running", true); - SenderError firstErr = newSenderError(); - SenderError secondErr = newSenderError(); - LineSenderServerException first = new LineSenderServerException(firstErr); - LineSenderServerException second = new LineSenderServerException(secondErr); - - invokeRecordFatal(loop, first); - invokeRecordFatal(loop, second); - - // Only the first failure latches — subsequent calls must not - // overwrite, otherwise a follow-on cascade would mask the original - // root cause. - Assert.assertSame("first throwable must remain latched", - first, loop.getTerminalError()); - Assert.assertSame("first SenderError must remain latched", - firstErr, loop.getLastTerminalServerError()); - } - - private static SenderError newSenderError() { - return new SenderError( - SenderError.Category.SCHEMA_MISMATCH, - SenderError.Policy.TERMINAL, - 0x03, - "test-msg", - 7L, - 100L, 100L, - "tbl", - System.nanoTime() - ); - } - - private static CursorWebSocketSendLoop newBareLoop() throws Exception { - // Bypass the real constructor — we don't need a wire client or engine - // to test the latched-error contract. - return (CursorWebSocketSendLoop) Unsafe.getUnsafe() - .allocateInstance(CursorWebSocketSendLoop.class); - } - - private static void setField(Object target, String name, Object value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.set(target, value); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - private static void invokeRecordFatal(CursorWebSocketSendLoop loop, Throwable t) - throws Exception { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( - "recordFatal", Throwable.class); - m.setAccessible(true); - m.invoke(loop, t); - } -} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java index 130f60b44..4a8b156f1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopJvmErrorTest.java @@ -25,33 +25,68 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; -import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; /** * Regression coverage (M3): {@code catch (Throwable)} in the reconnect * machinery used to swallow {@link java.lang.Error} (OOM, LinkageError, * StackOverflowError) into an indefinite "transport outage" retry with only * a throttled, possibly-null-message WARN as a trace. A JVM/programming - * failure is not a transport outage -- retrying cannot clear it -- so every - * retry loop must rethrow {@code Error}, after latching it as terminal where - * a producer could otherwise hang in {@code checkError()}. - *

      - * Uses the same {@code Unsafe.allocateInstance} bare-loop pattern as - * {@link CursorWebSocketSendLoopErrorLatchTest}: the retry loops only touch - * the fields wired below, so no live wire client or engine is needed. + * failure is not a transport outage -- retrying cannot clear it -- so the + * blocking connection helper must rethrow {@code Error} immediately and the + * background loop must latch it for the producer before stopping. */ public class CursorWebSocketSendLoopJvmErrorTest { + @Test(timeout = 30_000) + public void testIoLoopLatchesReconnectJvmErrorAndShutsDown() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + LinkageError injected = new LinkageError("simulated JVM failure"); + try (CursorSendEngine engine = new CursorSendEngine(null, 64 * 1024)) { + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + attempts.incrementAndGet(); + throw injected; + }, + 1L, + 4L, + false + ); + try { + loop.start(); + long deadlineNanos = System.nanoTime() + 5_000_000_000L; + while ((loop.getTerminalError() == null || loop.isRunning()) + && System.nanoTime() < deadlineNanos) { + Thread.yield(); + } + + Throwable terminal = loop.getTerminalError(); + Assert.assertNotNull("I/O loop did not latch the JVM Error", terminal); + Assert.assertEquals("a JVM Error must not be retried", 1, attempts.get()); + Assert.assertFalse("a terminal JVM Error must stop the loop", loop.isRunning()); + try { + loop.checkError(); + Assert.fail("producer-facing checkError() must surface the terminal failure"); + } catch (LineSenderException thrown) { + Assert.assertSame("checkError() must rethrow the latched wrapper", terminal, thrown); + Assert.assertSame("the wrapper must retain the original JVM Error", injected, thrown.getCause()); + } + } finally { + // Also proves that the real I/O-thread shutdown latch is released. + loop.close(); + } + } + } + @Test public void testConnectWithRetryPropagatesJvmError() { // The budgeted blocking initial-connect helper must not burn the @@ -75,110 +110,4 @@ public void testConnectWithRetryPropagatesJvmError() { Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); } - @Test - public void testConnectLoopPropagatesJvmErrorAndLatchesTerminal() throws Exception { - // The background per-outage reconnect loop must (1) latch the Error - // as terminal FIRST -- a producer parked in checkError() would - // otherwise never observe the failure -- and (2) rethrow so the I/O - // thread dies loudly instead of reconnect-looping forever. - CursorWebSocketSendLoop loop = newBareLoop(); - AtomicInteger attempts = new AtomicInteger(); - wireReconnectPlumbing(loop, attempts); - - Method connectLoop = CursorWebSocketSendLoop.class.getDeclaredMethod( - "connectLoop", Throwable.class, String.class, long.class); - connectLoop.setAccessible(true); - try { - connectLoop.invoke(loop, new LineSenderException("initial wire failure"), "reconnect", 0L); - Assert.fail("a JVM Error must escape connectLoop, not be retried"); - } catch (InvocationTargetException ite) { - Assert.assertTrue("expected LinkageError, got " + ite.getCause(), - ite.getCause() instanceof LinkageError); - } - Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); - assertErrorLatchedAndStopped(loop); - } - - @Test - public void testIoLoopDoesNotFunnelJvmErrorIntoReconnect() throws Exception { - // ioLoop's catch (Throwable) used to funnel EVERYTHING into - // fail(t) -> connectLoop(t, "reconnect"). An Error must instead be - // latched as terminal and rethrown; the finally still counts down - // the shutdown latch so close() cannot hang on the dead thread. - CursorWebSocketSendLoop loop = newBareLoop(); - AtomicInteger attempts = new AtomicInteger(); - wireReconnectPlumbing(loop, attempts); - CountDownLatch shutdownLatch = new CountDownLatch(1); - setField(loop, "shutdownLatch", shutdownLatch); - // client == null + running routes ioLoop into attemptInitialConnect - // -> connectLoop -> the throwing factory, exercising the full funnel. - - Method ioLoop = CursorWebSocketSendLoop.class.getDeclaredMethod("ioLoop"); - ioLoop.setAccessible(true); - try { - ioLoop.invoke(loop); - Assert.fail("a JVM Error must escape ioLoop, not re-enter the reconnect loop"); - } catch (InvocationTargetException ite) { - Assert.assertTrue("expected LinkageError, got " + ite.getCause(), - ite.getCause() instanceof LinkageError); - } - Assert.assertEquals("no retry on a JVM Error", 1, attempts.get()); - Assert.assertEquals("shutdown latch must count down so close() cannot hang", - 0L, shutdownLatch.getCount()); - assertErrorLatchedAndStopped(loop); - } - - private static void assertErrorLatchedAndStopped(CursorWebSocketSendLoop loop) - throws Exception { - Throwable terminal = loop.getTerminalError(); - Assert.assertNotNull("Error must be latched as terminal for checkError()", terminal); - Assert.assertTrue("latch wraps the raw Error once", - terminal instanceof LineSenderException); - Assert.assertTrue("latched cause must be the original Error", - terminal.getCause() instanceof LinkageError); - Assert.assertFalse("recordFatal must stop the loop", - (Boolean) getField(loop, "running")); - try { - loop.checkError(); - Assert.fail("producer-facing checkError must surface the latched terminal"); - } catch (LineSenderException thrown) { - Assert.assertSame(terminal, thrown); - } - } - - /** - * Wires the minimal state the reconnect paths dereference: a factory - * throwing {@link LinkageError} on every attempt, live {@code running}, - * and the attempt counters (field initializers do not run under - * {@code Unsafe.allocateInstance}). - */ - private static void wireReconnectPlumbing(CursorWebSocketSendLoop loop, - AtomicInteger attempts) throws Exception { - CursorWebSocketSendLoop.ReconnectFactory factory = () -> { - attempts.incrementAndGet(); - throw new LinkageError("simulated JVM failure"); - }; - setField(loop, "reconnectFactory", factory); - setField(loop, "running", true); - setField(loop, "totalReconnectAttempts", new AtomicLong()); - setField(loop, "totalReconnects", new AtomicLong()); - } - - private static CursorWebSocketSendLoop newBareLoop() throws Exception { - // Bypass the real constructor -- no wire client or engine needed. - return (CursorWebSocketSendLoop) Unsafe.getUnsafe() - .allocateInstance(CursorWebSocketSendLoop.class); - } - - private static Object getField(Object target, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.get(target); - } - - private static void setField(Object target, String name, Object value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.set(target, value); - } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java new file mode 100644 index 000000000..d59ddd8f4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/QwpSchemaReplayNetworkTest.java @@ -0,0 +1,840 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpSchemaCapabilityMismatchException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import io.questdb.client.test.tools.TestUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.LockSupport; + +public class QwpSchemaReplayNetworkTest { + private static final int SEGMENT_SIZE = 1 << 20; + @Rule + public final TemporaryFolder temp = TemporaryFolder.builder().parentFolder(taskTempRoot()).build(); + + @Test + public void testBackgroundDrainerRetiresOrphanSchemaTailWithoutConnecting() throws Exception { + TestUtils.assertMemoryLeak(() -> { + File root = temp.newFolder("drainer-orphan-schema-tail"); + String slot = new File(root, "ghost").getAbsolutePath(); + seedDeferredSchemaTail(slot); + RecordingHandler oldHandler = new RecordingHandler(false, "deferred"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + IgnoringFactory factory = new IgnoringFactory(oldPeer.getPort(), 1); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, SEGMENT_SIZE, 4L * SEGMENT_SIZE, factory, + 5_000, 1, 4, false, 0 + ); + Thread thread = new Thread(drainer::run, "schema-orphan-tail-drainer"); + thread.start(); + try { + thread.join(5_000); + Assert.assertFalse("drainer did not finish", thread.isAlive()); + } finally { + drainer.requestStop(); + thread.join(5_000); + } + // The aborted tail is retired locally, so the drainer never needs a peer. + Assert.assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + Assert.assertEquals(0, factory.attempts.get()); + Assert.assertEquals(0, oldHandler.frames.size()); + } + assertNoQuarantine(root); + }); + } + + @Test + public void testBackgroundDrainerRejectsUnconfirmedClientReturnedByFactoryWithoutQuarantine() throws Exception { + File root = temp.newFolder("drainer-old-peer"); + String slot = new File(root, "ghost").getAbsolutePath(); + seedMixed(slot); + RecordingHandler oldHandler = new RecordingHandler(false, "replay"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + IgnoringFactory factory = new IgnoringFactory(oldPeer.getPort(), 2); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, SEGMENT_SIZE, 4L * SEGMENT_SIZE, factory, + 5_000, 1, 4, false, 0 + ); + AtomicInteger errors = new AtomicInteger(); + drainer.setErrorSink(error -> errors.incrementAndGet()); + Thread thread = new Thread(drainer::run, "schema-replay-drainer"); + thread.start(); + try { + Assert.assertTrue("factory must return at least two unconfirmed clients", + factory.awaitAttempts(5, TimeUnit.SECONDS)); + } finally { + drainer.requestStop(); + thread.join(5_000); + } + Assert.assertFalse("drainer did not stop", thread.isAlive()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.STOPPED, drainer.outcome()); + Assert.assertEquals(0, oldHandler.frames.size()); + Assert.assertEquals(0, errors.get()); + } + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test(timeout = 20_000) + public void testAckedSchemaFramesDoNotBlockLegacyPeerAfterDowngrade() throws Exception { + TestUtils.assertMemoryLeak(() -> { + File root = temp.newFolder("acked-then-downgrade"); + String slot = new File(root, "default").getAbsolutePath(); + RecordingHandler supportingHandler = new RecordingHandler(true, "downgrade"); + RecordingHandler legacyHandler = new RecordingHandler(true, "downgrade"); + try (TestWebSocketServer legacyPeer = new TestWebSocketServer(legacyHandler); + CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer schemaRows = table("downgrade", 1); + QwpTableBuffer legacyRows = table("downgrade", 2)) { + legacyPeer.start(); + Assert.assertTrue(legacyPeer.awaitStart(5, TimeUnit.SECONDS)); + SchemaAwareFactory factory = new SchemaAwareFactory(legacyPeer.getPort(), 1); + CursorWebSocketSendLoop loop = null; + try { + try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { + supportingPeer.setAdvertiseSchema(true); + supportingPeer.start(); + Assert.assertTrue(supportingPeer.awaitStart(5, TimeUnit.SECONDS)); + WebSocketClient initialClient = connectLegacyClient(supportingPeer.getPort(), true); + loop = new CursorWebSocketSendLoop( + initialClient, engine, 0, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, 1, 4, false); + Assert.assertTrue(initialClient.isQwpSchemaEnabled()); + loop.start(); + int length = encoder.encodeSchema(schemaRows, -1, -1); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + Assert.assertTrue(engine.requiresSchema()); + awaitAckedFsn(engine, 0); + Assert.assertFalse("an acked schema frame must not pin the capability", + engine.requiresSchema()); + } + // The schema-capable peer is gone; the loop must accept the legacy peer. + int length = encoder.encode(legacyRows); + Assert.assertEquals(1, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + awaitAckedFsn(engine, 1); + Assert.assertNull(loop.getTerminalError()); + Assert.assertEquals(1, supportingHandler.frames.size()); + Assert.assertEquals(1, legacyHandler.frames.size()); + Assert.assertEquals(0, legacyHandler.frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] + & QwpConstants.FLAG_SCHEMA); + } finally { + if (loop != null) { + loop.close(); + } + } + } + }); + } + + @Test + public void testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames() throws Exception { + File root = temp.newFolder("mixed"); + String slot = new File(root, "default").getAbsolutePath(); + byte[][] expected = seedMixed(slot); + + RecordingHandler oldHandler = new RecordingHandler(false, "replay"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + Assert.assertThrows(QwpSchemaCapabilityMismatchException.class, + () -> Sender.fromConfig(config(oldPeer.getPort(), root, true))); + Assert.assertEquals(0, oldHandler.frames.size()); + } + assertRetained(slot, 1); + assertNoQuarantine(root); + + RecordingHandler supportingHandler = new RecordingHandler(true, "replay"); + try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { + supportingPeer.setAdvertiseSchema(true); + supportingPeer.start(); + Assert.assertTrue(supportingPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(supportingPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals(2, supportingHandler.frames.size()); + Assert.assertArrayEquals(expected[0], supportingHandler.frames.get(0)); + Assert.assertArrayEquals(expected[1], supportingHandler.frames.get(1)); + assertNoQuarantine(root); + } + + @Test(timeout = 15_000) + public void testOffReplaysRecoveredSchemaFramesInForegroundAndOrphan() throws Exception { + TestUtils.assertMemoryLeak(() -> { + for (boolean isOrphan : new boolean[]{false, true}) { + File root = temp.newFolder(isOrphan ? "off-orphan" : "off-foreground"); + String slot = new File(root, isOrphan ? "ghost" : "default").getAbsolutePath(); + byte[][] expected = seedMixed(slot); + RecordingHandler handler = new RecordingHandler(true, "replay"); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + if (isOrphan) { + // The owner itself has no schema backlog; only this drainer opts in. + try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig( + "ws::addr=localhost:" + peer.getPort() + ";schema_mode=off;")) { + Assert.assertFalse(peer.hasRequestedSchema()); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, SEGMENT_SIZE, 4L * SEGMENT_SIZE, + sender.newBackgroundReconnectFactory(() -> false), + 5_000, 1, 4, false, 0); + Thread thread = new Thread(drainer, "off-schema-orphan"); + thread.start(); + try { + thread.join(5_000); + Assert.assertFalse("drainer did not finish", thread.isAlive()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + } finally { + drainer.requestStop(); + thread.join(5_000); + } + } + } else { + try (Sender sender = Sender.fromConfig(config(peer.getPort(), root, false) + + "schema_mode=off;")) { + Assert.assertTrue(sender.drain(5_000)); + } + } + Assert.assertTrue(peer.hasRequestedSchema()); + } + Assert.assertEquals(2, handler.frames.size()); + Assert.assertArrayEquals(expected[0], handler.frames.get(0)); + Assert.assertArrayEquals(expected[1], handler.frames.get(1)); + assertNoQuarantine(root); + } + }); + } + + @Test + public void testPublicSenderSchemaFrameSurvivesRestartByteForByte() throws Exception { + File root = temp.newFolder("public-sender-restart"); + String slot = new File(root, "default").getAbsolutePath(); + SchemaRecordingHandler firstHandler = new SchemaRecordingHandler(false); + byte[] expected; + try (TestWebSocketServer firstPeer = new TestWebSocketServer(firstHandler)) { + firstPeer.setAdvertiseSchema(true); + firstPeer.start(); + Assert.assertTrue(firstPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(firstPeer.getPort(), root, false))) { + sender.table("small_replay").byteColumn("value", (byte) -7).atNow(); + Assert.assertEquals(0, sender.flushAndGetSequence()); + expected = firstHandler.awaitDataFrame(); + } + } + assertRetained(slot, 0); + + SchemaRecordingHandler replayHandler = new SchemaRecordingHandler(true); + try (TestWebSocketServer replayPeer = new TestWebSocketServer(replayHandler)) { + replayPeer.setAdvertiseSchema(true); + replayPeer.start(); + Assert.assertTrue(replayPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(replayPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertArrayEquals(expected, replayHandler.awaitDataFrame()); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + expected[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + assertNoQuarantine(root); + } + + @Test + public void testDeferredOnlySchemaTailIsAbortedLocallyBeforeConnectingToOldPeer() throws Exception { + File root = temp.newFolder("deferred-old-peer"); + String slot = new File(root, "default").getAbsolutePath(); + seedDeferredSchemaTail(slot); + + RecordingHandler oldHandler = new RecordingHandler(true, "deferred"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(oldPeer.getPort(), root, false))) { + sender.table("deferred").longColumn("n", 2).atNow(); + sender.flush(); + Assert.assertTrue(sender.drain(10_000)); + } + Assert.assertEquals("only the new legacy row may reach the old peer", 1, oldHandler.frames.size()); + Assert.assertEquals(0, oldHandler.frames.get(0)[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + } + try (CursorSendEngine recovered = new CursorSendEngine(slot, SEGMENT_SIZE)) { + Assert.assertFalse(recovered.requiresSchema()); + Assert.assertEquals(recovered.publishedFsn(), recovered.ackedFsn()); + } + assertNoQuarantine(root); + } + + @Test + public void testDeferredOnlySchemaTailIsAbortedLocallyBeforeConnectingToSupportingPeer() throws Exception { + File root = temp.newFolder("deferred"); + String slot = new File(root, "default").getAbsolutePath(); + seedDeferredSchemaTail(slot); + + RecordingHandler supportingHandler = new RecordingHandler(true, "deferred"); + try (TestWebSocketServer supportingPeer = new TestWebSocketServer(supportingHandler)) { + supportingPeer.setAdvertiseSchema(true); + supportingPeer.start(); + Assert.assertTrue(supportingPeer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(supportingPeer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals("an uncommitted recovered tail must be aborted, not transmitted", 0, supportingHandler.frames.size()); + assertNoQuarantine(root); + } + + @Test + public void testLiveSchemaAppendIsRetainedWhenConnectedPeerIsLegacyOnly() throws Exception { + File root = temp.newFolder("live-append-old-peer"); + String slot = new File(root, "default").getAbsolutePath(); + RecordingHandler oldHandler = new RecordingHandler(false, "live_append"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + WebSocketClient initialClient = connectLegacyClient(oldPeer.getPort(), false); + SchemaAwareFactory factory = new SchemaAwareFactory(oldPeer.getPort(), 2); + CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, engine, 0, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, 1, 4, false); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer table = table("live_append", 7)) { + loop.start(); + int length = encoder.encodeSchema(table, -1, -1); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + Assert.assertTrue("schema-required reconnects must remain bounded and observable", + factory.awaitAttempts(5, TimeUnit.SECONDS)); + Assert.assertTrue(loop.isRunning()); + Assert.assertNull(loop.getTerminalError()); + Assert.assertEquals("no extended frame may reach a legacy peer", 0, oldHandler.frames.size()); + Assert.assertEquals(-1, engine.ackedFsn()); + Assert.assertEquals(0, engine.publishedFsn()); + } finally { + loop.close(); + engine.close(); + } + } + assertRetained(slot, 0); + assertNoQuarantine(root); + } + + @Test + public void testSchemaFeedbackAckAdvancesRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-ack"); + seedMixed(new File(root, "default").getAbsolutePath()); + FeedbackHandler handler = new FeedbackHandler(false); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + try (Sender sender = Sender.fromConfig(config(peer.getPort(), root, false))) { + Assert.assertTrue(sender.drain(10_000)); + } + } + Assert.assertEquals(2, handler.frames.get()); + assertNoQuarantine(root); + } + + @Test + public void testMalformedSchemaFeedbackDoesNotAdvanceRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-malformed"); + String slot = new File(root, "default").getAbsolutePath(); + seedMixed(slot); + FeedbackHandler handler = new FeedbackHandler(true); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + Sender sender = Sender.fromConfig(config(peer.getPort(), root, false)); + try { + Assert.assertTrue("malformed feedback must be parsed, rejected, and trigger replay", + handler.awaitDistinctClients(5, TimeUnit.SECONDS)); + } finally { + sender.close(); + } + } + Assert.assertTrue(handler.frames.get() > 0); + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test + public void testInvalidSchemaColumnsDoNotAdvanceRecoveredBacklog() throws Exception { + File root = temp.newFolder("feedback-invalid-columns"); + String slot = new File(root, "default").getAbsolutePath(); + seedMixed(slot); + FeedbackHandler handler = new FeedbackHandler(false, true); + try (TestWebSocketServer peer = new TestWebSocketServer(handler)) { + peer.setAdvertiseSchema(true); + peer.start(); + Assert.assertTrue(peer.awaitStart(5, TimeUnit.SECONDS)); + Sender sender = Sender.fromConfig(config(peer.getPort(), root, false)); + try { + Assert.assertTrue("invalid schema feedback must be rejected and replayed", + handler.awaitDistinctClients(5, TimeUnit.SECONDS)); + } finally { + sender.close(); + } + } + Assert.assertTrue(handler.frames.get() > 0); + assertRetained(slot, 1); + assertNoQuarantine(root); + } + + @Test + public void testSchemaRequirementIsIsolatedAcrossForegroundAndBackgroundFactories() throws Exception { + TestUtils.assertMemoryLeak(() -> { + RecordingHandler oldHandler = new RecordingHandler(false, "unused"); + try (TestWebSocketServer oldPeer = new TestWebSocketServer(oldHandler)) { + oldPeer.start(); + Assert.assertTrue(oldPeer.awaitStart(5, TimeUnit.SECONDS)); + assertSchemaRequirementIsolated(oldPeer.getPort(), true); + assertSchemaRequirementIsolated(oldPeer.getPort(), false); + Assert.assertEquals(0, oldHandler.frames.size()); + } + }); + } + + private static void assertNoQuarantine(File root) { + File[] files = root.listFiles(); + Assert.assertNotNull(files); + for (File file : files) { + Assert.assertFalse(file.getName(), file.getName().contains(".unreplayable-")); + Assert.assertFalse(file.getName(), file.getName().equals(OrphanScanner.FAILED_SENTINEL_NAME)); + File failed = new File(file, OrphanScanner.FAILED_SENTINEL_NAME); + Assert.assertFalse(failed.getAbsolutePath(), failed.exists()); + } + } + + private static WebSocketClient connectLegacyClient(int port, boolean requestSchema) { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + if (requestSchema) { + client.requestQwpSchema(); + } + client.upgrade("/write/v4", 5_000, null); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + private static void assertRetained(String slot, long publishedFsn) { + try (CursorSendEngine recovered = new CursorSendEngine(slot, SEGMENT_SIZE)) { + Assert.assertTrue(recovered.requiresSchema()); + Assert.assertEquals(-1, recovered.ackedFsn()); + Assert.assertEquals(publishedFsn, recovered.publishedFsn()); + } + } + + private static void assertSchemaRequirementIsolated(int port, boolean foregroundFirst) throws Exception { + try (QwpWebSocketSender sender = (QwpWebSocketSender) Sender.fromConfig( + "ws::addr=localhost:" + port + ';')) { + CursorWebSocketSendLoop.ReconnectFactory foreground; + CursorWebSocketSendLoop.ReconnectFactory background; + if (foregroundFirst) { + foreground = sender.newReconnectFactory(); + background = sender.newBackgroundReconnectFactory(() -> false); + foreground.setSchemaRequired(true); + assertSchemaMismatch(foreground); + try (WebSocketClient client = background.reconnect()) { + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } else { + background = sender.newBackgroundReconnectFactory(() -> false); + foreground = sender.newReconnectFactory(); + background.setSchemaRequired(true); + assertSchemaMismatch(background); + try (WebSocketClient client = foreground.reconnect()) { + Assert.assertFalse(client.isQwpSchemaEnabled()); + } + } + } + } + + private static void assertSchemaMismatch(CursorWebSocketSendLoop.ReconnectFactory factory) throws Exception { + WebSocketClient client = null; + try { + client = factory.reconnect(); + Assert.fail("an old peer must be rejected by the schema-required factory"); + } catch (QwpSchemaCapabilityMismatchException expected) { + // Expected: the requirement belongs to this factory's stream. + } finally { + if (client != null) { + client.close(); + } + } + } + + private static void awaitAckedFsn(CursorSendEngine engine, long fsn) { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (engine.ackedFsn() < fsn) { + if (System.nanoTime() > deadline) { + Assert.fail("timed out waiting for ackedFsn " + fsn + ", actual " + engine.ackedFsn()); + } + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(1)); + } + } + + private static String config(int port, File root, boolean durableAck) { + return "ws::addr=localhost:" + port + + ";sf_dir=" + root.getAbsolutePath() + + ";sender_id=default;sf_max_segment_bytes=1m;request_durable_ack=" + + (durableAck ? "on" : "off") + ";close_flush_timeout_millis=0;"; + } + + private static byte[] copy(QwpWebSocketEncoder encoder, int length) { + byte[] bytes = new byte[length]; + long address = encoder.getBuffer().getBufferPtr(); + for (int i = 0; i < length; i++) { + bytes[i] = io.questdb.client.std.Unsafe.getUnsafe().getByte(address + i); + } + return bytes; + } + + private static byte[][] seedMixed(String slot) { + try (CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer legacy = table("replay", 1); + QwpTableBuffer schema = table("replay", 2)) { + byte[] first = copy(encoder, encoder.encode(legacy)); + Assert.assertEquals(0, first[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), first.length)); + byte[] second = copy(encoder, encoder.encodeSchema(schema, -1, -1)); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, second[QwpConstants.HEADER_OFFSET_FLAGS] & QwpConstants.FLAG_SCHEMA); + Assert.assertEquals(1, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), second.length)); + return new byte[][]{first, second}; + } + } + + private static void seedDeferredSchemaTail(String slot) { + try (CursorSendEngine engine = new CursorSendEngine(slot, SEGMENT_SIZE); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer table = table("deferred", 1)) { + encoder.setDeferCommit(true); + int length = encoder.encodeSchema(table, -1, -1); + Assert.assertEquals(0, engine.appendBlocking(encoder.getBuffer().getBufferPtr(), length)); + } + } + + private static QwpTableBuffer table(String name, long value) { + QwpTableBuffer table = new QwpTableBuffer(name); + table.getOrCreateColumn("n", QwpConstants.TYPE_LONG, true).addLong(value); + table.nextRow(); + return table; + } + + private static File taskTempRoot() { + File root = new File("target/schema-replay-network").getAbsoluteFile(); + Assert.assertTrue(root.exists() || root.mkdirs()); + return root; + } + + private static final class RecordingHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledge; + private final List frames = new ArrayList<>(); + private final String tableName; + + private RecordingHandler(boolean acknowledge, String tableName) { + this.acknowledge = acknowledge; + this.tableName = tableName; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + int sequence = frames.size(); + frames.add(data); + if (acknowledge) { + try { + client.sendBinary(ok(sequence)); + client.sendBinary(durable(sequence)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private byte[] durable(long seqTxn) { + byte[] name = tableName.getBytes(java.nio.charset.StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.allocate(1 + 2 + 2 + name.length + 8).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(WebSocketResponse.STATUS_DURABLE_ACK).putShort((short) 1).putShort((short) name.length).put(name).putLong(seqTxn); + return buffer.array(); + } + + private static byte[] ok(long sequence) { + return ByteBuffer.allocate(11).order(ByteOrder.LITTLE_ENDIAN) + .put(WebSocketResponse.STATUS_OK).putLong(sequence).putShort((short) 0).array(); + } + } + + private static final class SchemaRecordingHandler implements TestWebSocketServer.WebSocketServerHandler { + private final boolean acknowledge; + private final List dataFrames = new ArrayList<>(); + + private SchemaRecordingHandler(boolean acknowledge) { + this.acknowledge = acknowledge; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (data.length >= QwpConstants.HEADER_SIZE + && data[QwpConstants.HEADER_OFFSET_FLAGS] == QwpSchemaProtocol.FLAG_CONTROL) { + ByteBuffer request = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + sendSchema(client, request.getLong(QwpConstants.HEADER_SIZE + 1)); + return; + } + dataFrames.add(data); + notifyAll(); + if (acknowledge) { + try { + client.sendBinary(RecordingHandler.ok(0)); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private synchronized byte[] awaitDataFrame() throws InterruptedException { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (dataFrames.isEmpty()) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) { + Assert.fail("timed out waiting for schema data frame"); + } + TimeUnit.NANOSECONDS.timedWait(this, remaining); + } + return dataFrames.get(0); + } + + private static void sendSchema(TestWebSocketServer.ClientHandler client, long requestId) { + byte[] value = "value".getBytes(StandardCharsets.UTF_8); + byte[] timestamp = "ts".getBytes(StandardCharsets.UTF_8); + int payloadLength = 1 + Long.BYTES + 1 + Integer.BYTES + Long.BYTES + + Short.BYTES + Short.BYTES + + Short.BYTES + value.length + Integer.BYTES + Short.BYTES + + Short.BYTES + timestamp.length + Integer.BYTES + Short.BYTES; + ByteBuffer response = ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + response.putInt(QwpConstants.MAGIC_MESSAGE) + .put((byte) QwpConstants.VERSION) + .put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0) + .putInt(payloadLength) + .put(QwpSchemaProtocol.KIND_SCHEMA) + .putLong(requestId) + .put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(301) + .putLong(401) + .putShort((short) 1) + .putShort((short) 2) + .putShort((short) value.length) + .put(value) + .putInt(ColumnType.INT) + .putShort((short) 0) + .putShort((short) timestamp.length) + .put(timestamp) + .putInt(ColumnType.TIMESTAMP_NANO) + .putShort((short) 0); + try { + client.sendBinary(response.array()); + } catch (IOException e) { + throw new AssertionError(e); + } + } + } + + private static final class IgnoringFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final CountDownLatch attemptsLatch; + private final int port; + + private IgnoringFactory(int port, int expectedAttempts) { + this.port = port; + this.attemptsLatch = new CountDownLatch(expectedAttempts); + } + + boolean awaitAttempts(long timeout, TimeUnit unit) throws InterruptedException { + return attemptsLatch.await(timeout, unit); + } + + @Override + public WebSocketClient reconnect() { + attempts.incrementAndGet(); + attemptsLatch.countDown(); + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + boolean success = false; + try { + client.connect("localhost", port); + client.upgrade("/write/v4", 5_000, null); + Assert.assertFalse(client.isQwpSchemaEnabled()); + success = true; + return client; + } finally { + if (!success) { + client.close(); + } + } + } + + @Override + public void setSchemaRequired(boolean isRequired) { + // Deliberately ignored: returned-client validation must still reject it. + } + } + + private static final class FeedbackHandler implements TestWebSocketServer.WebSocketServerHandler { + private final AtomicInteger frames = new AtomicInteger(); + private final CountDownLatch distinctClients; + private final boolean malformed; + private final boolean invalidColumns; + private TestWebSocketServer.ClientHandler lastClient; + + private FeedbackHandler(boolean malformed) { + this(malformed, false); + } + + private FeedbackHandler(boolean malformed, boolean invalidColumns) { + this.malformed = malformed; + this.invalidColumns = invalidColumns; + this.distinctClients = new CountDownLatch(malformed || invalidColumns ? 2 : 1); + } + + private boolean awaitDistinctClients(long timeout, TimeUnit unit) throws InterruptedException { + return distinctClients.await(timeout, unit); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + int sequence = frames.getAndIncrement(); + byte[] response = schemaFeedbackOk(sequence); + if (malformed) { + response = java.util.Arrays.copyOf(response, response.length - 1); + } else if (invalidColumns) { + response = schemaFeedbackInvalidColumns(sequence); + } + try { + client.sendBinary(response); + if (client != lastClient) { + lastClient = client; + distinctClients.countDown(); + } + } catch (IOException e) { + throw new AssertionError(e); + } + } + + private static byte[] schemaFeedbackOk(long sequence) { + byte[] tableName = "replay".getBytes(StandardCharsets.UTF_8); + ByteBuffer buffer = ByteBuffer.allocate(11 + 2 + 2 + tableName.length + 4 + 10) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES) + .putLong(sequence).putShort((short) 0).putShort((short) 1) + .putShort((short) tableName.length).put(tableName).putInt(10) + .put((byte) 2).putLong(0).put((byte) 1); + return buffer.array(); + } + + private static byte[] schemaFeedbackInvalidColumns(long sequence) { + byte[] tableName = "replay".getBytes(StandardCharsets.UTF_8); + byte[] column = "x".getBytes(StandardCharsets.UTF_8); + int payloadLength = 26 + 2 * (2 + column.length + 6); + ByteBuffer buffer = ByteBuffer.allocate(11 + 2 + 2 + tableName.length + 4 + payloadLength) + .order(ByteOrder.LITTLE_ENDIAN); + buffer.put((byte) WebSocketResponse.SCHEMA_FEEDBACK_MODE_UPDATES) + .putLong(sequence).putShort((short) 0).putShort((short) 1) + .putShort((short) tableName.length).put(tableName).putInt(payloadLength) + .put((byte) 2).putLong(0).put((byte) 0).putInt(1).putLong(1) + .putShort((short) -1).putShort((short) 2); + for (byte name : new byte[]{'x', 'X'}) { + buffer.putShort((short) 1).put(name).putInt(5).putShort((short) 0); + } + return buffer.array(); + } + } + + private static final class SchemaAwareFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final CountDownLatch attemptsLatch; + private final int port; + private volatile boolean isSchemaRequired; + + private SchemaAwareFactory(int port, int expectedAttempts) { + this.port = port; + this.attemptsLatch = new CountDownLatch(expectedAttempts); + } + + boolean awaitAttempts(long timeout, TimeUnit unit) throws InterruptedException { + return attemptsLatch.await(timeout, unit); + } + + @Override + public WebSocketClient reconnect() { + attemptsLatch.countDown(); + return connectLegacyClient(port, isSchemaRequired); + } + + @Override + public void setSchemaRequired(boolean isRequired) { + isSchemaRequired = isRequired; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java index d6ec8151b..ce61cb122 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java @@ -25,8 +25,11 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import org.junit.Assert; @@ -35,6 +38,7 @@ import org.junit.rules.TemporaryFolder; import java.nio.file.Path; +import java.util.concurrent.TimeUnit; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -63,6 +67,105 @@ public void testUnterminatedSymbolLengthMarksRecoveredDeltaAsGap() throws Except assertMalformedDeltaMarksGap(new byte[]{0, 1, (byte) 0x80}); } + @Test + public void testSchemaRequirementRecoveredFromUnackedExtendedFrame() throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-schema-recovery").toPath(); + long payload = qwpFrame(QwpConstants.FLAG_SCHEMA); + try { + try (CursorSendEngine writer = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertFalse(writer.requiresSchema()); + Assert.assertEquals(0, writer.appendBlocking(payload, frameLength(payload))); + Assert.assertTrue(writer.requiresSchema()); + } + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertTrue(recovered.requiresSchema()); + } + } finally { + Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testAcknowledgedExtendedResidueDoesNotRequireSchema() throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-schema-acked-residue").toPath(); + long payload = qwpFrame(QwpConstants.FLAG_SCHEMA); + long legacy = qwpFrame((byte) 0); + try { + try (CursorSendEngine writer = new CursorSendEngine( + slot.toString(), 4_096, Long.MAX_VALUE, 30_000_000_000L, 1L)) { + Assert.assertEquals(0, writer.appendBlocking(payload, frameLength(payload))); + Assert.assertEquals(1, writer.appendBlocking(legacy, frameLength(legacy))); + Assert.assertTrue(writer.acknowledge(0)); + Assert.assertEquals(2, writer.appendBlocking(legacy, frameLength(legacy))); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try (AckWatermark watermark = AckWatermark.open(slot.toString())) { + if (watermark.read() >= 0) { + break; + } + } + Assert.assertTrue("ACK watermark was not checkpointed", System.nanoTime() < deadline); + Thread.yield(); + } + } + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertEquals(0, recovered.ackedFsn()); + Assert.assertEquals(2, recovered.publishedFsn()); + Assert.assertNotNull("acknowledged schema frame must remain represented", + recovered.findSegmentContaining(0)); + Assert.assertFalse(recovered.requiresSchema()); + } + } finally { + Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); + Unsafe.free(legacy, frameLength(legacy), MemoryTag.NATIVE_DEFAULT); + } + }); + } + + @Test + public void testExtendedDeferredTailRetiresLocallyAndDropsSchemaRequirement() throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-schema-tail").toPath(); + long payload = qwpFrame((byte) (QwpConstants.FLAG_SCHEMA | QwpConstants.FLAG_DEFER_COMMIT)); + try { + try (CursorSendEngine writer = new CursorSendEngine(slot.toString(), 4_096)) { + writer.appendBlocking(payload, frameLength(payload)); + } + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertTrue(recovered.requiresSchema()); + // Retirement transmits nothing, so it needs no schema-capable peer. + Assert.assertTrue(recovered.retireRecoveredOrphanTailIfReady()); + Assert.assertEquals(0, recovered.ackedFsn()); + Assert.assertFalse(recovered.requiresSchema()); + } + } finally { + Unsafe.free(payload, frameLength(payload), MemoryTag.NATIVE_DEFAULT); + } + }); + } + + private static long qwpFrame(byte flags) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer table = new QwpTableBuffer("schema_table")) { + table.getOrCreateColumn("x", QwpConstants.TYPE_LONG, true).addLong(1); + table.nextRow(); + encoder.setDeferCommit((flags & QwpConstants.FLAG_DEFER_COMMIT) != 0); + int length = (flags & QwpConstants.FLAG_SCHEMA) != 0 + ? encoder.encodeSchema(table, 1, 1) + : encoder.encode(table); + long payload = Unsafe.malloc(length, MemoryTag.NATIVE_DEFAULT); + io.questdb.client.std.Vect.memcpy(payload, encoder.getBuffer().getBufferPtr(), length); + return payload; + } + } + + private static int frameLength(long address) { + return QwpConstants.HEADER_SIZE + Unsafe.getUnsafe().getInt(address + 8); + } + private void assertMalformedDeltaMarksGap(byte[] deltaSection) throws Exception { assertMemoryLeak(() -> { Path slot = temporaryFolder.newFolder("qwp-malformed-recovery").toPath(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java index ace6d4f15..a4cea0ded 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpConstantsTest.java @@ -46,6 +46,7 @@ public void testGetFixedTypeSize() { Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_SHORT)); Assert.assertEquals(2, QwpConstants.getFixedTypeSize(TYPE_CHAR)); Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_INT)); + Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_IPv4)); Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_LONG)); Assert.assertEquals(4, QwpConstants.getFixedTypeSize(TYPE_FLOAT)); Assert.assertEquals(8, QwpConstants.getFixedTypeSize(TYPE_DOUBLE)); @@ -67,6 +68,7 @@ public void testGetFixedTypeSize() { public void testGetTypeName() { Assert.assertEquals("BOOLEAN", QwpConstants.getTypeName(TYPE_BOOLEAN)); Assert.assertEquals("INT", QwpConstants.getTypeName(TYPE_INT)); + Assert.assertEquals("IPv4", QwpConstants.getTypeName(TYPE_IPv4)); Assert.assertEquals("VARCHAR", QwpConstants.getTypeName(TYPE_VARCHAR)); Assert.assertEquals("TIMESTAMP", QwpConstants.getTypeName(TYPE_TIMESTAMP)); Assert.assertEquals("TIMESTAMP_NANOS", QwpConstants.getTypeName(TYPE_TIMESTAMP_NANOS)); @@ -94,6 +96,7 @@ public void testIsFixedWidthType() { Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_SHORT)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_CHAR)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_INT)); + Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_IPv4)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_LONG)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_FLOAT)); Assert.assertTrue(QwpConstants.isFixedWidthType(TYPE_DOUBLE)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java new file mode 100644 index 000000000..beaf1de4d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBenchmark.java @@ -0,0 +1,127 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +/** + * Per-row cost of writing an exact-type row through {@link QwpSchemaBinding} + * versus the legacy {@link QwpTableBuffer#getOrCreateColumn} path. Both write + * the same 10 columns (5 LONG, 5 DOUBLE) in the same order, so the difference + * is the binding's per-value column resolution. Run with {@link #main}. + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 2) +@Fork(1) +public class QwpSchemaBindingBenchmark { + private static final int COLUMN_COUNT = 10; + private static final int ROWS_PER_BATCH = 1_000; + private final String[] names = new String[COLUMN_COUNT]; + private QwpSchemaBinding binding; + private QwpTableBuffer legacyBuffer; + @Param({"2", "14"}) + private int nameLength; + private QwpTableBuffer schemaBuffer; + private long value; + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(QwpSchemaBindingBenchmark.class.getSimpleName()) + .build(); + new Runner(opt).run(); + } + + @Benchmark + public void legacyRow() { + long v = value++; + for (int i = 0; i < COLUMN_COUNT; i += 2) { + legacyBuffer.getOrCreateColumn(names[i], QwpConstants.TYPE_LONG, true).addLong(v); + legacyBuffer.getOrCreateColumn(names[i + 1], QwpConstants.TYPE_DOUBLE, true).addDouble(v); + } + legacyBuffer.nextRow(); + if (legacyBuffer.getRowCount() == ROWS_PER_BATCH) { + legacyBuffer.reset(); + } + } + + @Benchmark + public void schemaRow() { + long v = value++; + for (int i = 0; i < COLUMN_COUNT; i += 2) { + binding.longColumn(names[i], v); + binding.doubleColumn(names[i + 1], v); + } + schemaBuffer.nextRow(); + if (schemaBuffer.getRowCount() == ROWS_PER_BATCH) { + schemaBuffer.reset(); + } + } + + @Setup(Level.Trial) + public void setUp() { + byte[][] columns = new byte[COLUMN_COUNT][]; + for (int i = 0; i < COLUMN_COUNT; i++) { + StringBuilder name = new StringBuilder().append((char) ('a' + i)); + while (name.length() < nameLength) { + name.append('x'); + } + names[i] = name.toString(); + columns[i] = QwpSchemaTestFixtures.column(names[i], i % 2 == 0 ? ColumnType.LONG : ColumnType.DOUBLE); + } + legacyBuffer = new QwpTableBuffer("t"); + schemaBuffer = new QwpTableBuffer("t"); + binding = QwpSchemaTestFixtures.binding(schemaBuffer, columns); + } + + @TearDown(Level.Trial) + public void tearDown() { + legacyBuffer.close(); + schemaBuffer.close(); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java new file mode 100644 index 000000000..2e46eef5b --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBinaryTest.java @@ -0,0 +1,426 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.bytes.DirectByteSlice; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingBinaryTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/binary-input.tsv"; + + @Test + public void testConformanceCorpusUsesExactBinaryWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBinaryTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertEquals(line, "BINARY", fields[3]); + Assert.assertEquals(line, "VALID", fields[6]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + if ("BINARY".equals(fields[1])) { + binding.binaryColumn("value", bytes(fields[2])); + } else if ("STRING".equals(fields[1])) { + binding.stringColumn("value", "".equals(fields[2]) ? null : utf16(fields[2])); + } else { + throw new AssertionError(fields[1]); + } + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + if ("".equals(fields[4])) { + Assert.assertEquals(fields[0], 1, reader.u8()); + Assert.assertEquals(fields[0], 1, reader.u8()); + Assert.assertEquals(fields[0], 0, reader.i32()); + } else { + byte[] expected = bytes(fields[4]); + if ("binary_all_256".equals(fields[0])) { + assertAllByteValues(bytes(fields[2])); + assertAllByteValues(expected); + assertAllByteValues(bytes(fields[5])); + } + Assert.assertEquals(fields[0], 0, reader.u8()); + Assert.assertEquals(fields[0], 0, reader.i32()); + Assert.assertEquals(fields[0], expected.length, reader.i32()); + Assert.assertArrayEquals(fields[0], expected, reader.bytes(expected.length)); + } + Assert.assertEquals(fields[0], encoder.getBuffer().getPosition(), reader.position()); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testAllBinaryOverloadsCopyBeforeReturning() throws Exception { + assertMemoryLeak(() -> { + long address = Unsafe.malloc(6, MemoryTag.NATIVE_DEFAULT); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + byte[] array = {1, 2}; + binding.binaryColumn("value", array); + array[0] = 9; + buffer.nextRow(); + + put(address, new byte[]{3, 4}); + DirectByteSlice slice = new DirectByteSlice().of(address, 2); + binding.binaryColumn("value", slice); + put(address, new byte[]{8, 8}); + slice.of(0, 0); + buffer.nextRow(); + + put(address, new byte[]{5, 6}); + binding.binaryColumn("value", address, 2); + put(address, new byte[]{7, 7}); + buffer.nextRow(); + Unsafe.free(address, 6, MemoryTag.NATIVE_DEFAULT); + address = 0; + binding.binaryColumn("value", 0, 0); + buffer.nextRow(); + + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 4, 1); + Assert.assertEquals(0, reader.u8()); + for (int offset : new int[]{0, 2, 4, 6, 6}) { + Assert.assertEquals(offset, reader.i32()); + } + Assert.assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6}, reader.bytes(6)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } finally { + if (address != 0) { + Unsafe.free(address, 6, MemoryTag.NATIVE_DEFAULT); + } + } + }); + } + + @Test + public void testDuplicatePrecedesBinaryArgumentValidation() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.binaryColumn("value", new byte[]{42}) + .binaryColumn("value", (byte[]) null) + .binaryColumn("value", (DirectByteSlice) null) + .binaryColumn("value", 0, 1) + .binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1) + .stringColumn("value", new ThrowingCharSequence(new TestCharSequenceException())); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(42, reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.stringColumn("value", "x").binaryColumn("value", (byte[]) null); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals('x', reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testInvalidBinaryArgumentsAreTypedAndCheckedBeforeMemoryAccess() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + assertInvalid("binary value", () -> binding.binaryColumn("value", (byte[]) null)); + rollback(buffer); + assertInvalid("binary slice", () -> binding.binaryColumn("value", (DirectByteSlice) null)); + rollback(buffer); + assertInvalid("non-negative", () -> binding.binaryColumn("value", new DirectByteSlice().of(1, -1))); + rollback(buffer); + assertInvalid("pointer", () -> binding.binaryColumn("value", new DirectByteSlice().of(0, 1))); + rollback(buffer); + assertInvalid("non-negative", () -> binding.binaryColumn("value", 1, -1)); + rollback(buffer); + assertInvalid("pointer", () -> binding.binaryColumn("value", 0, 1)); + rollback(buffer); + assertInvalid("maximum", () -> binding.binaryColumn("value", 1, (long) Integer.MAX_VALUE + 1)); + } + } + + @Test + public void testBitmapCrossesSixtyFourRowsWithPresentAndOmittedValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + for (int row = 0; row < 130; row++) { + if ((row & 1) == 0) { + binding.binaryColumn("value", new byte[]{(byte) row}); + } + buffer.nextRow(); + } + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 130, 1); + Assert.assertEquals(1, reader.u8()); + for (int i = 0; i < 16; i++) { + Assert.assertEquals(0xaa, reader.u8()); + } + Assert.assertEquals(0x02, reader.u8()); + for (int i = 0; i <= 65; i++) { + Assert.assertEquals(i, reader.i32()); + } + for (int row = 0; row < 130; row += 2) { + Assert.assertEquals(row & 0xff, reader.u8()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.BINARY), column("existing_b", ColumnType.BINARY), + column("only_b", ColumnType.BINARY), column("c", ColumnType.BINARY)); + binding.binaryColumn("a", new byte[]{1}); + binding.binaryColumn("existing_b", new byte[]{2}); + buffer.nextRow(); + TestCharSequenceException existingFailure = new TestCharSequenceException(); + Assert.assertSame(existingFailure, Assert.assertThrows(TestCharSequenceException.class, + () -> binding.stringColumn("existing_b", new ThrowingCharSequence(existingFailure)))); + rollback(buffer); + TestCharSequenceException newColumnFailure = new TestCharSequenceException(); + Assert.assertSame(newColumnFailure, Assert.assertThrows(TestCharSequenceException.class, + () -> binding.stringColumn("only_b", new ThrowingCharSequence(newColumnFailure)))); + rollback(buffer); + binding.binaryColumn("c", new byte[]{3}); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 3); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); + Assert.assertEquals("existing_b", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BINARY, reader.u8()); + assertSparseBinary(reader, 2, 1); + assertSparseBinary(reader, 2, 2); + assertSparseBinary(reader, 1, 3); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testStringAppendIsExceptionAtomicForVarcharRuntimeAndError() throws Exception { + assertMemoryLeak(() -> { + for (Throwable failure : new Throwable[]{new TestCharSequenceException(), new TestCharSequenceError()}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.VARCHAR)); + binding.stringColumn("value", "a"); + buffer.nextRow(); + ThrowingCharSequence input = failure instanceof Error + ? new ThrowingCharSequence(failure, 4096, 300) + : new ThrowingCharSequence(failure); + Throwable actual = Assert.assertThrows(failure.getClass(), + () -> binding.stringColumn("value", input)); + Assert.assertSame(failure, actual); + rollback(buffer); + binding.stringColumn("value", "c"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + for (int offset : new int[]{0, 1, 2}) { + Assert.assertEquals(offset, reader.i32()); + } + Assert.assertArrayEquals(new byte[]{'a', 'c'}, reader.bytes(2)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } + }); + } + + @Test + public void testResetClearAndSchemaGuards() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BINARY)); + binding.binaryColumn("value", new byte[]{7}); + buffer.nextRow(); + buffer.reset(); + binding.binaryColumn("value", new byte[]{8}); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(8, reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + Assert.assertThrows(IllegalStateException.class, () -> binding.binaryColumn("value", new byte[]{1})); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.STRING))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.INT))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, -1, column("value", ColumnType.BINARY, new byte[]{1}))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + assertUnsupported(binding(buffer, 0, column("value", ColumnType.BINARY))); + } + }); + } + + private static void assertSparseBinary(QwpTestWireReader reader, int bitmap, int value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(value, reader.u8()); + } + + private static void assertAllByteValues(byte[] bytes) { + Assert.assertEquals(256, bytes.length); + for (int i = 0; i < bytes.length; i++) { + Assert.assertEquals(i, bytes[i] & 0xff); + } + } + + private static void assertUnsupported(QwpSchemaBinding binding) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> binding.binaryColumn("value", new byte[]{1})); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, error.getReason()); + } + + private static void assertInvalid(String detail, Runnable runnable) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(error.getMessage(), LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("table=t")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=BINARY")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=BINARY")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains(detail)); + } + + private static void rollback(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static void put(long address, byte[] bytes) { + for (int i = 0; i < bytes.length; i++) { + Unsafe.getUnsafe().putByte(address + i, bytes[i]); + } + } + + private static byte[] bytes(String hex) { + Assert.assertEquals(hex, 0, hex.length() & 1); + byte[] result = new byte[hex.length() / 2]; + for (int i = 0; i < result.length; i++) { + result[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16); + } + return result; + } + + private static String utf16(String hex) { + Assert.assertEquals(hex, 0, hex.length() & 3); + char[] chars = new char[hex.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + return tableReader(encoder, size, rows, columns, QwpConstants.TYPE_BINARY); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, int columns, int wireType) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + return reader; + } + + private static final class ThrowingCharSequence implements CharSequence { + private final Throwable failure; + private final int failureIndex; + private final int length; + + private ThrowingCharSequence(Throwable failure) { + this(failure, 2, 1); + } + + private ThrowingCharSequence(Throwable failure, int length, int failureIndex) { + this.failure = failure; + this.length = length; + this.failureIndex = failureIndex; + } + + @Override + public int length() { + return length; + } + + @Override + public char charAt(int index) { + if (index < failureIndex) { + return 'x'; + } + if (failure instanceof Error) { + throw (Error) failure; + } + throw (RuntimeException) failure; + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + } + + private static final class TestCharSequenceException extends RuntimeException { + } + + private static final class TestCharSequenceError extends Error { + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java new file mode 100644 index 000000000..1e0136c89 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingBooleanTest.java @@ -0,0 +1,310 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingBooleanTest { + private static final String BOOLEAN_CORPUS = "/io/questdb/client/cutlass/qwp/boolean-to-target.tsv"; + private static final String STRING_CORPUS = "/io/questdb/client/cutlass/qwp/string-to-boolean.tsv"; + + @Test + public void testBooleanCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBooleanTest.class.getResourceAsStream(BOOLEAN_CORPUS); + Assert.assertNotNull(BOOLEAN_CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 6, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(fields[2]))); + Assert.assertTrue(fields[1], "TRUE".equals(fields[1]) || "FALSE".equals(fields[1])); + binding.boolColumn("value", "TRUE".equals(fields[1])); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(fields[3]), 1); + Assert.assertEquals(0, reader.u8()); + assertValue(reader, fields[3], fields[4]); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testStringCorpusParsesStrictAsciiAndUsesBooleanWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingBooleanTest.class.getResourceAsStream(STRING_CORPUS); + Assert.assertNotNull(STRING_CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 4, fields.length); + CharSequence value = "".equals(fields[1]) ? null : utf16(fields[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BOOLEAN)); + if ("".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("value", value)); + } else { + binding.stringColumn("value", value); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 1); + if (value == null) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(fields[2]), reader.u8()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(80, count); + }); + } + + @Test + public void testBooleanPackingPastEightAndSixtyFourWithOmissions() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.BOOLEAN)); + for (int row = 0; row < 137; row++) { + if ((row & 1) == 0) { + binding.boolColumn("value", (row & 3) == 0); + } + buffer.nextRow(); + } + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 137); + Assert.assertEquals(1, reader.u8()); + byte[] nulls = new byte[18]; + for (int i = 0; i < 17; i++) nulls[i] = (byte) 0xaa; + Assert.assertArrayEquals(nulls, reader.bytes(nulls.length)); + byte[] expected = new byte[9]; + for (int i = 0; i < 8; i++) expected[i] = 0x55; + expected[8] = 0x15; + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + + buffer.reset(); + for (int row = 0; row < 70; row++) buffer.nextRow(); + QwpTestWireReader allNull = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_BOOLEAN, 70); + Assert.assertEquals(1, allNull.u8()); + byte[] allNulls = new byte[9]; + for (int i = 0; i < 8; i++) allNulls[i] = (byte) 0xff; + allNulls[8] = 0x3f; + Assert.assertArrayEquals(allNulls, allNull.bytes(allNulls.length)); + Assert.assertEquals(encoder.getBuffer().getPosition(), allNull.position()); + } + }); + } + + @Test + public void testDuplicateRollbackFailedRowAndLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.BOOLEAN), column("bad", ColumnType.BOOLEAN), column("c", ColumnType.INT)); + binding.boolColumn("a", true); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "invalid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.boolColumn("c", true); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + StringBuilder mutable = new StringBuilder("TrUe"); + binding.stringColumn("a", mutable).boolColumn("a", false); + mutable.replace(0, mutable.length(), "false"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reset, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + buffer.clear(); + assertIllegalState(() -> binding.boolColumn("a", true)); + assertIllegalState(() -> binding.stringColumn("a", "true")); + } + }); + } + + @Test + public void testDuplicatePrecedesUnsupportedCrossSetterAndInvalidParse() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("flag", ColumnType.BOOLEAN), column("sym", ColumnType.SYMBOL)); + binding.boolColumn("flag", true).stringColumn("flag", "invalid"); + binding.stringColumn("sym", "first").boolColumn("sym", false); + buffer.nextRow(); + Assert.assertTrue(encoder.encodeSchema(buffer) > 0); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("sym", ColumnType.SYMBOL), column("future", ColumnType.BOOLEAN, new byte[]{1}), + column("flagged", ColumnType.BOOLEAN | 0x10000), column("plain", ColumnType.BOOLEAN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("sym", true)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.symbol("plain", null)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.longColumn("plain", Long.MIN_VALUE)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("future", true)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("flagged", "true")); + rollback(buffer); + binding.boolColumn("missing", true); + Assert.assertEquals(QwpConstants.TYPE_BOOLEAN, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.BOOLEAN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.boolColumn("d", true)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("d", null)); + } + } + + private static void assertValue(QwpTestWireReader reader, String type, String expected) { + switch (type) { + case "BOOLEAN": Assert.assertEquals(Integer.parseInt(expected), reader.u8()); break; + case "BYTE": Assert.assertEquals(Integer.parseInt(expected), reader.u8()); break; + case "SHORT": Assert.assertEquals(Integer.parseInt(expected), reader.i16()); break; + case "INT": Assert.assertEquals(Integer.parseInt(expected), reader.i32()); break; + case "LONG": Assert.assertEquals(Long.parseLong(expected), reader.i64()); break; + case "FLOAT": Assert.assertEquals(Float.floatToRawIntBits(Float.parseFloat(expected)), Float.floatToRawIntBits(reader.f32())); break; + case "DOUBLE": Assert.assertEquals(Double.doubleToRawLongBits(Double.parseDouble(expected)), Double.doubleToRawLongBits(reader.f64())); break; + case "VARCHAR": + byte[] bytes = hex(expected); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(bytes.length, reader.i32()); + Assert.assertArrayEquals(bytes, reader.bytes(bytes.length)); + break; + default: throw new AssertionError("unknown wire type: " + type); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "BOOLEAN": return ColumnType.BOOLEAN; + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + case "STRING": return ColumnType.STRING; + case "VARCHAR": return ColumnType.VARCHAR; + default: throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BOOLEAN": return QwpConstants.TYPE_BOOLEAN; + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + case "VARCHAR": return QwpConstants.TYPE_VARCHAR; + default: throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } + + private static String utf16(String value) { + Assert.assertEquals(0, value.length() & 3); + char[] chars = new char[value.length() / 4]; + for (int i = 0; i < chars.length; i++) chars[i] = (char) Integer.parseInt(value.substring(i * 4, i * 4 + 4), 16); + return new String(chars); + } + + private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } + private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java new file mode 100644 index 000000000..aaff94b8a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingCharTest.java @@ -0,0 +1,211 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; + +public class QwpSchemaBindingCharTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/char-conversions.tsv"; + private static final String HEADER = "# case_id\tinput_kind\tutf16_hex\texpected_char\texpected_null"; + + @Test + public void testCharToTextUsesTargetNativeUtf8() { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", targetType))); + binding.charColumn("value", 'A'); + buffer.nextRow(); + binding.charColumn("value", '\u03a9'); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(3, reader.i32()); + Assert.assertEquals('A', reader.u8()); + Assert.assertEquals(0xce, reader.u8()); + Assert.assertEquals(0xa9, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + } + } + + @Test + public void testCorpusUsesExactNativeCharWire() throws Exception { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean headerSeen = false; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + Assert.assertTrue("true".equals(fields[4]) || "false".equals(fields[4])); + Assert.assertEquals(Integer.parseInt(fields[3]) == 0, Boolean.parseBoolean(fields[4])); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + if ("CHAR".equals(fields[1])) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", ColumnType.CHAR))); + String input = fromUtf16Hex(fields[2]); + Assert.assertEquals(1, input.length()); + binding.charColumn("value", input.charAt(0)); + } else if ("STRING".equals(fields[1])) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", ColumnType.CHAR))); + binding.stringColumn("value", "".equals(fields[2]) ? null : fromUtf16Hex(fields[2])); + } else { + throw new AssertionError("unknown input kind: " + fields[1]); + } + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_CHAR); + if ("".equals(fields[2])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(fields[3]), reader.u16()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(23, count); + } + + @Test + public void testDuplicateOmissionRollbackResetAndInference() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, + column("a", ColumnType.CHAR), column("bad", ColumnType.UUID), column("c", ColumnType.CHAR))); + binding.stringColumn("a", "A").binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "B"); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("bad", 'C')); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "Z"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 3, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(6, reader.u8()); + Assert.assertEquals('A', reader.u16()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals('Z', reader.u16()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "Q"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_CHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals('Q', reset.u16()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "A"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.charColumn("value", 'A'); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_CHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + } + + @Test + public void testUnsupportedAndDesignatedTargetsRemainRejected() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, + column("value", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("value", 'A')); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.charColumn("ts", 'A')); + } + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(-1, column("value", targetType))); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.charColumn("value", '\ud800')); + } + } + } + + private static String fromUtf16Hex(String hex) { + StringBuilder value = new StringBuilder(hex.length() / 4); + for (int i = 0; i < hex.length(); i += 4) { + value.append((char) Integer.parseInt(hex.substring(i, i + 4), 16)); + } + return value.toString(); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java new file mode 100644 index 000000000..4afa3669a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingColumnResolutionTest.java @@ -0,0 +1,155 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * A binding resolves each column against the schema once, when it creates the + * buffer column, and later writes reuse that resolution. These tests pin that + * the reuse keeps every per-value rule of the first write. + */ +public class QwpSchemaBindingColumnResolutionTest { + + @Test + public void testDuplicateWriteToResolvedColumnKeepsFirstValue() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 1); + buffer.nextRow(); + binding.longColumn("v", 2); + binding.longColumn("V", 3); + buffer.nextRow(); + assertInts(buffer.getColumn(0), 1, 2); + } + }); + } + + @Test + public void testInferredColumnTypeConflictIsDetectedOnLaterRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("x", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleColumn("x", 2.0), "inferred column type conflict"); + buffer.cancelCurrentRow(); + binding.longColumn("X", 3); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + QwpTableBuffer.ColumnBuffer column = buffer.getColumn(0); + Assert.assertEquals(QwpConstants.TYPE_LONG, column.getType()); + Assert.assertEquals(2, column.getSize()); + Assert.assertEquals(1, Unsafe.getUnsafe().getLong(column.getDataAddress())); + Assert.assertEquals(3, Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES)); + } + }); + } + + @Test + public void testInvalidNameIsRejectedAfterColumnsExist() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("", 2), "invalid column name"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn(null, 2), "invalid column name"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("v?", 2), "invalid column name"); + Assert.assertEquals(1, buffer.getColumnCount()); + } + }); + } + + @Test + public void testRejectedSchemaColumnsStayRejectedAcrossRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, + column("ts", ColumnType.TIMESTAMP_MICRO), + parameterizedColumn("p", ColumnType.LONG), + column("v", ColumnType.INT)); + for (int attempt = 0; attempt < 2; attempt++) { + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("ts", 1), "designated timestamp writes"); + buffer.cancelCurrentRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("p", 1), "parameterized target type"); + buffer.cancelCurrentRow(); + binding.longColumn("v", attempt); + buffer.nextRow(); + } + Assert.assertEquals("rejected targets must not create buffer columns", 1, buffer.getColumnCount()); + assertInts(buffer.getColumn(0), 0, 1); + } + }); + } + + @Test + public void testResolvedColumnKeepsConversionAndRangeChecks() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.INT)); + binding.longColumn("v", 7); + buffer.nextRow(); + binding.longColumn("V", 8); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("v", 1L << 40)); + buffer.cancelCurrentRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.binaryColumn("v", new byte[]{1}), "conversion is not implemented"); + buffer.cancelCurrentRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + Assert.assertEquals(QwpConstants.TYPE_INT, buffer.getColumn(0).getType()); + assertInts(buffer.getColumn(0), 7, 8); + } + }); + } + + private static void assertInts(QwpTableBuffer.ColumnBuffer column, int... expected) { + Assert.assertEquals(expected.length, column.getSize()); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals(expected[i], Unsafe.getUnsafe().getInt(column.getDataAddress() + (long) i * Integer.BYTES)); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java new file mode 100644 index 000000000..cc69a925c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingDoubleArrayTest.java @@ -0,0 +1,273 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.line.array.DoubleArray; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingDoubleArrayTest { + + @Test + public void testAllPublicRepresentationsUseExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + int[] shape32 = new int[ColumnType.ARRAY_NDIMS_LIMIT]; + java.util.Arrays.fill(shape32, 1); + double payloadNaN = Double.longBitsToDouble(0x7ff8000000000042L); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray array = new DoubleArray(shape32)) { + array.append(10.5); + QwpSchemaBinding binding = binding(buffer, + column("a1", arrayType(1)), + column("a2", arrayType(2)), + column("a3", arrayType(3)), + column("a32", arrayType(ColumnType.ARRAY_NDIMS_LIMIT))); + binding.doubleArray("a1", new double[]{payloadNaN, -0.0, Double.POSITIVE_INFINITY}); + binding.doubleArray("a2", new double[][]{{3.0, 4.0}, {5.0, 6.0}}); + binding.doubleArray("a3", new double[][][]{{{7.0}, {8.0}}}); + binding.doubleArray("a32", array); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 4); + columnDef(reader, "a1"); + columnDef(reader, "a2"); + columnDef(reader, "a3"); + columnDef(reader, "a32"); + array(reader, new int[]{3}, payloadNaN, -0.0, Double.POSITIVE_INFINITY); + array(reader, new int[]{2, 2}, 3.0, 4.0, 5.0, 6.0); + array(reader, new int[]{1, 2, 1}, 7.0, 8.0); + array(reader, shape32, 10.5); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNullRepresentationsAreNoopsAndWrapperRankTracksReshape() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray array = new DoubleArray(1, 1)) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.doubleArray("a", (double[]) null) + .doubleArray("b", (double[][]) null) + .doubleArray("c", (double[][][]) null) + .doubleArray("d", (DoubleArray) null); + Assert.assertEquals(0, buffer.getColumnDefs().length); + + Assert.assertEquals(2, array.getDimensionality()); + array.reshape(1); + Assert.assertEquals(1, array.getDimensionality()); + array.append(42.0); + binding.doubleArray("value", array); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1, -1, -1); + columnDef(reader, "value"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 1, new int[]{1}, new double[]{42.0}); + Assert.assertEquals(size, reader.position()); + } + }); + + try (DoubleArray array = new DoubleArray(1)) { + array.close(); + Assert.assertThrows(LineSenderException.class, array::getDimensionality); + } + } + + @Test + public void testKnownRankDuplicateRollbackAndShapeValidation() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t"); + DoubleArray closedDuplicate = new DoubleArray(1)) { + QwpSchemaBinding binding = binding(buffer, + column("v", arrayType(1)), column("m", arrayType(2))); + + binding.doubleArray("v", new double[]{1.0}); + binding.doubleArray("v", new double[][]{{99.0}}); // duplicate: first value wins + closedDuplicate.close(); + binding.doubleArray("v", closedDuplicate); // duplicate: do not inspect an ignored value + buffer.nextRow(); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[][]{{2.0}}), + "sourceDims=2", "targetDims=1"); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + Assert.assertThrows(LineSenderException.class, + () -> binding.doubleArray("m", new double[][]{{2.0}, {3.0, 4.0}})); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.doubleArray("v", new double[0]); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1); + columnDef(reader, "v"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 1, new int[]{1}, new double[]{1.0}); + rawArray(reader, 1, new int[]{0}, new double[0]); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testMissingSchemaPinsFirstEffectiveRank() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.doubleArray("v", new double[][]{{1.0, 2.0}}); + buffer.nextRow(); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{3.0}), + "sourceDims=1", "inferredDims=2"); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.doubleArray("v", new double[][]{{4.0}, {5.0}}); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 1, -1, -1); + columnDef(reader, "v"); + Assert.assertEquals(0, reader.u8()); + rawArray(reader, 2, new int[]{1, 2}, new double[]{1.0, 2.0}); + rawArray(reader, 2, new int[]{2, 1}, new double[]{4.0, 5.0}); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{6.0}), + "sourceDims=1", "inferredDims=2"); + } + }); + } + + @Test + public void testRejectsWrongElementTypeWeakRankParametersAndDesignatedColumn() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("v", ColumnType.encodeArrayType(ColumnType.LONG, 1))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "targetType=LONG[]"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + int weak = arrayType(1) | (1 << 19); + QwpSchemaBinding binding = binding(buffer, column("v", weak)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "conversion is not implemented"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("v", arrayType(1), new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "parameterized target type"); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("v", arrayType(1))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleArray("v", new double[]{1.0}), "designated timestamp writes"); + } + } + + @Test + public void testRejectsStringValuesForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("a", "zz")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("a", "")); + } + } + }); + } + + @Test + public void testRejectsGeoHashValuesForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.geoHashColumn("a", 1023L, 10)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.geoHashColumn("a", "zz")); + } + } + }); + } + + @Test + public void testStringNullUsesArrayNullWireForAllArrayRanks() throws Exception { + assertMemoryLeak(() -> { + for (int dimensions = 1; dimensions <= ColumnType.ARRAY_NDIMS_LIMIT; dimensions++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("a", arrayType(dimensions))); + binding.stringColumn("a", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + columnDef(reader, "a"); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + private static void array(QwpTestWireReader reader, int[] shape, double... values) { + Assert.assertEquals(0, reader.u8()); + rawArray(reader, shape.length, shape, values); + } + + private static int arrayType(int dimensions) { + return ColumnType.encodeArrayType(ColumnType.DOUBLE, dimensions); + } + + private static void columnDef(QwpTestWireReader reader, String name) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE_ARRAY, reader.u8()); + } + + private static void rawArray(QwpTestWireReader reader, int expectedDims, int[] shape, double[] values) { + Assert.assertEquals(expectedDims, reader.u8()); + for (int dimension : shape) { + Assert.assertEquals(dimension, reader.i32()); + } + for (double value : values) { + Assert.assertEquals(Double.doubleToRawLongBits(value), reader.i64()); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java new file mode 100644 index 000000000..0ba41eafd --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingDecimalTest.java @@ -0,0 +1,301 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv"; + private static final String HEADER = "# case_id\tsource_type\tinput_bits_hex\ttarget_type\t" + + "target_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\t" + + "expected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingFloatingDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[][] sourceTargets = new boolean[2][6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 12, fields.length); + Vector vector = new Vector(fields); + sourceTargets[vector.sourceIndex()][targetIndex(vector.targetType)] = true; + assertVector(vector); + count++; + } + } + Assert.assertEquals(60, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target pair " + source + '/' + target, + sourceTargets[source][target]); + } + } + }); + } + + @Test + public void testDuplicateNullRollbackAndOmissionLifecycle() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(3, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + // Duplicate suppression precedes both an invalid conversion and null normalization. + binding.floatColumn("value", 1.0f) + .doubleColumn("value", 1.25) + .doubleColumn("value", Double.NEGATIVE_INFINITY); + buffer.nextRow(); + + // A null first value also wins over a later invalid duplicate. + binding.doubleColumn("value", Double.longBitsToDouble(0x7ff8000000000042L)) + .floatColumn("value", 1.25f); + buffer.nextRow(); + + // Row B must disappear without disturbing completed A or following C. + binding.doubleColumn("only_b", 2.0); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.doubleColumn("value", 1.25)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.floatColumn("value", -2.5f); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_DECIMAL64, 4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0a, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-25, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL64, 1); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void assertInvalid(Vector vector) throws Exception { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> vector.append(binding, "value")); + Assert.assertEquals(vector.caseId, LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(vector.caseId, error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=value")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=" + vector.sourceType)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=DECIMAL")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } + } + + private static void assertVector(Vector vector) throws Exception { + try { + if (vector.invalid()) { + assertInvalid(vector); + return; + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + vector.append(binding, "value"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType), 1); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + return reader; + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + return 5; + } + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final long inputBits; + private final String outcome; + private final String sourceType; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + sourceType = fields[1]; + Assert.assertTrue(sourceType.equals("FLOAT") || sourceType.equals("DOUBLE")); + inputBits = Long.parseUnsignedLong(fields[2], 16); + Assert.assertEquals(sourceType.equals("FLOAT") ? 8 : 16, fields[2].length()); + targetType = fields[3]; + targetPrecision = Integer.parseInt(fields[4]); + targetScale = Integer.parseInt(fields[5]); + outcome = fields[6]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) { + expectedLimbs[i] = parseHex(fields[7 + i]); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[7 + i]); + } + if (outcome.equals("NULL")) { + Assert.assertEquals("NULL", fields[11]); + } else if (outcome.equals("INVALID")) { + Assert.assertEquals("-", fields[11]); + } else { + Assert.assertFalse(fields[11].isEmpty()); + Assert.assertNotEquals("-", fields[11]); + } + } + + private void append(QwpSchemaBinding binding, String column) { + if ("FLOAT".equals(sourceType)) { + binding.floatColumn(column, Float.intBitsToFloat((int) inputBits)); + } else { + binding.doubleColumn(column, Double.longBitsToDouble(inputBits)); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private int sourceIndex() { + return "FLOAT".equals(sourceType) ? 0 : 1; + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java new file mode 100644 index 000000000..d2f1e7801 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTest.java @@ -0,0 +1,265 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingFloatingTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] f = line.split("\\t", -1); + Assert.assertEquals(line, 7, f.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(f[3]))); + Runnable append = () -> append(binding, f[1], f[2]); + if ("".equals(f[5])) { + Assert.assertEquals(f[0], "", f[6]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + } else { + append.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[4]), 1, 1); + if ("".equals(f[5])) { + Assert.assertEquals(f[0], + "BYTE".equals(f[3]) || "SHORT".equals(f[3]) ? "0" : "NULL", f[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertArrayEquals(f[0], hex(f[5]), reader.bytes(hex(f[5]).length)); + } + Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(350, count); + }); + } + + @Test + public void testNaNIsMissingWithOtherNullAndPresentValues() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DOUBLE)); + binding.floatColumn("value", Float.intBitsToFloat(0x7fc12345)); + buffer.nextRow(); + buffer.nextRow(); + binding.doubleColumn("value", -0.0d); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_DOUBLE, 3, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals(0x8000000000000000L, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testDuplicatePrecedesInvalidAndUnsupportedConversion() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("n", ColumnType.LONG), column("uuid", ColumnType.UUID)); + binding.floatColumn("n", 1).doubleColumn("n", Double.POSITIVE_INFINITY); + binding.uuidColumn("uuid", 1, 2).floatColumn("uuid", Float.NaN); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 1, 2); + Assert.assertEquals("n", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals("uuid", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.FLOAT), column("only_b", ColumnType.DOUBLE), + column("bad", ColumnType.BYTE), column("c", ColumnType.LONG)); + binding.doubleColumn("a", 1.5); + buffer.nextRow(); + binding.floatColumn("only_b", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.doubleColumn("bad", 128)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.floatColumn("c", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_FLOAT, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(Float.floatToRawIntBits(1.5f), reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testResetClearAndStaleBinding() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.FLOAT)); + binding.doubleColumn("value", 9.25); + buffer.nextRow(); + buffer.reset(); + binding.floatColumn("value", -0.0f); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_FLOAT, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x80000000, reader.i32()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + assertIllegalState(() -> binding.floatColumn("value", 1)); + assertIllegalState(() -> binding.doubleColumn("value", 1)); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("uuid", ColumnType.UUID), column("future", ColumnType.FLOAT, new byte[]{1}), + column("flagged", ColumnType.DOUBLE | 0x10000), column("bool", ColumnType.BOOLEAN), + column("timestamp", ColumnType.TIMESTAMP), column("binary", ColumnType.BINARY), + column("byte", ColumnType.BYTE), column("short", ColumnType.SHORT)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("uuid", Float.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("future", Double.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("flagged", 1)); + rollback(buffer); + binding.doubleColumn("missing", 1); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("bool", Float.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("timestamp", Double.NaN)); + rollback(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("binary", Float.NaN)); + rollback(buffer); + assertInvalidContext(binding, buffer, "byte", true); + assertInvalidContext(binding, buffer, "short", false); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.DOUBLE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.floatColumn("d", Float.NaN)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.doubleColumn("d", Double.NaN)); + } + } + + private static void append(QwpSchemaBinding binding, String inputType, String bits) { + if ("FLOAT".equals(inputType)) { + binding.floatColumn("value", Float.intBitsToFloat((int) Long.parseLong(bits, 16))); + } else if ("DOUBLE".equals(inputType)) { + binding.doubleColumn("value", Double.longBitsToDouble(Long.parseUnsignedLong(bits, 16))); + } else { + throw new AssertionError("unknown input type: " + inputType); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "BYTE": return ColumnType.BYTE; case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BYTE": return QwpConstants.TYPE_BYTE; case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + Assert.assertEquals(value, 0, value.length() & 1); byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring((bytes.length - 1 - i) * 2, (bytes.length - i) * 2), 16); + return bytes; + } + + private static void rollback(QwpTableBuffer buffer) { buffer.cancelCurrentRow(); buffer.rollbackUncommittedColumns(); } + private static void assertInvalidContext(QwpSchemaBinding binding, QwpTableBuffer buffer, String column, boolean floatInput) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, + () -> { if (floatInput) binding.floatColumn(column, 128); else binding.doubleColumn(column, Double.POSITIVE_INFINITY); }); + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, error.getReason()); + Assert.assertFalse(error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=" + column)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=" + (floatInput ? "FLOAT" : "DOUBLE"))); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + column.toUpperCase())); + rollback(buffer); + } + private static void assertIllegalState(Runnable action) { Assert.assertThrows(IllegalStateException.class, action::run); } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java new file mode 100644 index 000000000..c305bf3f4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingFloatingTextTest.java @@ -0,0 +1,325 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingFloatingTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/floating-to-text.tsv"; + + @Test + public void testCorpusUsesExactTargetTextWire() throws Exception { + assertMemoryLeak( + () -> { + InputStream stream = + QwpSchemaBindingFloatingTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = + new BufferedReader( + new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + boolean symbol = "SYMBOL".equals(fields[3]); + QwpSchemaBinding binding = + binding(buffer, column("value", targetType(fields[3]))); + append(binding, fields[1], fields[2], "value"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = + tableReader( + encoder, + size, + symbol + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR, + 1); + boolean missing = "".equals(fields[5]); + Assert.assertEquals(missing, "".equals(fields[6])); + Assert.assertEquals(missing ? 1 : 0, reader.u8()); + if (missing) { + Assert.assertEquals(1, reader.u8()); + if (symbol) { + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + } + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals( + fields[6], + new String(expected, StandardCharsets.UTF_8)); + if (symbol) { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals( + expected, reader.bytes(expected.length)); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError( + "case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(48, count); + }); + } + + @Test + public void testNaNDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak( + () -> { + for (int target : + new int[] {ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = + target == ColumnType.SYMBOL + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR; + QwpSchemaBinding binding = binding(buffer, column("value", target)); + binding.floatColumn("value", Float.intBitsToFloat(0x7fc00002)) + .binaryColumn("value", new byte[] {1}); + buffer.nextRow(); + binding.doubleColumn("value", -0.0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("-0.0", reader.string()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(4, reader.i32()); + Assert.assertEquals("-0.0", reader.ascii(4)); + } + Assert.assertEquals(size, reader.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = + binding( + buffer, + column("a", ColumnType.STRING), + column("bad", ColumnType.UUID), + column("c", ColumnType.VARCHAR)); + binding.doubleColumn("a", 1e23); + buffer.nextRow(); + binding.floatColumn("a", 0.1f); + assertReason( + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.doubleColumn("bad", 2.0)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.floatColumn("c", Float.intBitsToFloat(1)); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarchar(reader, 6, "1.0E23"); + assertVarchar(reader, 5, "1.401298464324817E-45"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.doubleColumn("a", Double.NEGATIVE_INFINITY); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + assertVarchar(reset, 0, "-Infinity"); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testSymbolFormatterScratchIsCopied() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.doubleColumn("value", 42.0); + buffer.nextRow(); + binding.doubleColumn("value", 43.0); + buffer.nextRow(); + binding.doubleColumn("value", 42.0); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("42.0", reader.string()); + Assert.assertEquals("43.0", reader.string()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testNativeInferenceAndUnsupportedTargets() { + assertNative("FLOAT", "value", 0x80000000L); + assertNative("DOUBLE", "value", 0x8000000000000000L); + for (int target : new int[] {ColumnType.TIMESTAMP, ColumnType.UUID, ColumnType.BOOLEAN}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", target)); + assertReason( + LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> + binding.doubleColumn( + "value", Double.longBitsToDouble(0x7ff8000000000002L))); + } + } + } + + private static void assertNative(String inputType, String name, long bits) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + append( + binding, + inputType, + inputType.equals("FLOAT") + ? String.format("%08x", bits) + : String.format("%016x", bits), + name); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals( + inputType.equals("FLOAT") ? QwpConstants.TYPE_FLOAT : QwpConstants.TYPE_DOUBLE, + reader.u8()); + Assert.assertEquals(0, reader.u8()); + if (inputType.equals("FLOAT")) { + Assert.assertEquals((int) bits, reader.i32()); + } else { + Assert.assertEquals(bits, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + + private static void append(QwpSchemaBinding binding, String type, String bits, String name) { + if ("FLOAT".equals(type)) + binding.floatColumn(name, Float.intBitsToFloat((int) Long.parseUnsignedLong(bits, 16))); + else binding.doubleColumn(name, Double.longBitsToDouble(Long.parseUnsignedLong(bits, 16))); + } + + private static void assertVarchar(QwpTestWireReader reader, int bitmap, String expected) { + if (bitmap == 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + } + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length(), reader.i32()); + Assert.assertEquals(expected, reader.ascii(expected.length())); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) { + return ColumnType.STRING; + } + if ("VARCHAR".equals(value)) { + return ColumnType.VARCHAR; + } + if ("SYMBOL".equals(value)) { + return ColumnType.SYMBOL; + } + throw new AssertionError(value); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java new file mode 100644 index 000000000..be50e3e1c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongDecimalTest.java @@ -0,0 +1,246 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-decimal.tsv"; + private static final String HEADER = "# case_id\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[] targets = new boolean[6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 11, fields.length); + Vector vector = new Vector(fields); + targets[targetIndex(vector.targetType)] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + if (vector.invalid()) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("value", vector.input)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } else { + binding.longColumn("value", vector.input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType)); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(39, count); + for (int i = 0; i < targets.length; i++) { + Assert.assertTrue("missing target " + i, targets[i]); + } + }); + } + + @Test + public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(2, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + binding.longColumn("value", 1).longColumn("value", 10); + buffer.nextRow(); + + binding.longColumn("only_b", 1); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.longColumn("value", 10)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.longColumn("value", -1); + buffer.nextRow(); + binding.longColumn("value", Long.MIN_VALUE).longColumn("value", 1); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0c, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-10, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 1); + decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + return 5; + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final long input; + private final String outcome; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + input = Long.parseLong(fields[1]); + targetType = fields[2]; + targetPrecision = Integer.parseInt(fields[3]); + targetScale = Integer.parseInt(fields[4]); + outcome = fields[5]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) { + expectedLimbs[i] = parseHex(fields[6 + i]); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[6 + i]); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java new file mode 100644 index 000000000..383ebf4ba --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTextTest.java @@ -0,0 +1,287 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-text.tsv"; + + @Test + public void testCorpusUsesExactTextWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = wireType(fields[3]); + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); + binding.longColumn("value", Long.parseLong(fields[1])); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 1); + if ("".equals(fields[4])) { + Assert.assertEquals("", fields[5]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) Assert.assertEquals(0, reader.i32()); + else Assert.assertEquals(0, reader.varint()); + } else { + byte[] expected = hex(fields[4]); + Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals(0, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + } else { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(18, count); + }); + } + + @Test + public void testDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR, ColumnType.SYMBOL}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + byte wireType = targetType == ColumnType.SYMBOL + ? QwpConstants.TYPE_SYMBOL + : QwpConstants.TYPE_VARCHAR; + QwpSchemaBinding binding = binding(buffer, column("value", targetType)); + binding.longColumn("value", Long.MIN_VALUE).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.bytes(2)); + } else { + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + } + Assert.assertEquals(size, reader.position()); + } + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.STRING), column("bad", ColumnType.UUID), + column("c", ColumnType.VARCHAR), column("sym", ColumnType.SYMBOL)); + binding.longColumn("a", 10).stringColumn("a", "ignored"); + buffer.nextRow(); + binding.longColumn("sym", 7).longColumn("sym", Long.MIN_VALUE); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("bad", 20)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.longColumn("c", 30); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, QwpConstants.TYPE_VARCHAR, 3, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(6, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); + Assert.assertArrayEquals(new byte[]{'1', '0'}, reader.bytes(2)); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(2, reader.i32()); + Assert.assertArrayEquals(new byte[]{'3', '0'}, reader.bytes(2)); + Assert.assertEquals(size, reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.longColumn("value", 8); + buffer.nextRow(); + buffer.reset(); + binding.longColumn("value", 42); + buffer.nextRow(); + binding.longColumn("value", 43); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(2, reset.varint()); + Assert.assertArrayEquals(new byte[]{'4', '2'}, reset.stringBytes()); + Assert.assertArrayEquals(new byte[]{'4', '3'}, reset.stringBytes()); + Assert.assertEquals(0, reset.varint()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(0, reset.varint()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingSchemaInfersNativeLong() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.longColumn("value", 42); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnCount()); + Assert.assertEquals(QwpConstants.TYPE_LONG, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(42, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testOwnerBackedSymbolUsesGlobalDictionary() throws Exception { + assertMemoryLeak(() -> { + TestWebSocketServer.WebSocketServerHandler handler = new TestWebSocketServer.WebSocketServerHandler() { }; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender senderApi = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + QwpWebSocketSender sender = (QwpWebSocketSender) senderApi; + Assert.assertEquals(0, sender.getOrAddGlobalSymbol("prefix")); + GlobalSymbolDictionary mirror = new GlobalSymbolDictionary(); + Assert.assertEquals(0, mirror.getOrAddSymbol("prefix")); + try (QwpTableBuffer buffer = new QwpTableBuffer("t", sender); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + binding.longColumn("value", 42); + buffer.nextRow(); + binding.longColumn("value", 43); + buffer.nextRow(); + binding.longColumn("value", 42); + buffer.nextRow(); + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("42")); + Assert.assertEquals(2, sender.getOrAddGlobalSymbol("43")); + Assert.assertEquals(0, buffer.getColumn(0).getSymbolDictionarySize()); + mirror.getOrAddSymbol("42"); + mirror.getOrAddSymbol("43"); + encoder.beginSchemaMessage(1, mirror, -1, 2); + encoder.addSchemaTable(buffer, binding.getTableId(), binding.getMetadataVersion()); + int size = encoder.finishMessage(); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(3, reader.varint()); + Assert.assertArrayEquals("prefix".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("42".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("43".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 3); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + } + }); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows) { + return tableReader(encoder, size, wireType, rows, 1); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + if (columns == 1) { + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + } + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) return ColumnType.STRING; + if ("VARCHAR".equals(value)) return ColumnType.VARCHAR; + if ("SYMBOL".equals(value)) return ColumnType.SYMBOL; + throw new AssertionError(value); + } + + private static byte wireType(String value) { + return "SYMBOL".equals(value) ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR; + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java new file mode 100644 index 000000000..5cfa9e527 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingLongTimestampTest.java @@ -0,0 +1,186 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingLongTimestampTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv"; + + @Test + public void testCorpusUsesExactTargetUnitWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingLongTimestampTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + int target = targetType(fields[2]); + byte wireType = wireType(fields[3]); + long value = Long.parseLong(fields[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.longColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", wireType); + if ("".equals(fields[4])) { + Assert.assertEquals("true", fields[5]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals("false", fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(12, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + byte wireType = target == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.longColumn("value", Long.MIN_VALUE).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.longColumn("value", Long.MIN_VALUE + 1); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, "value", wireType); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.MIN_VALUE + 1, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.longColumn("value", Long.MAX_VALUE); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, 1, "value", wireType); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(Long.MAX_VALUE, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.TIMESTAMP_MICRO), + column("bad", ColumnType.UUID), + column("c", ColumnType.TIMESTAMP_NANO)); + binding.longColumn("a", -1); + buffer.nextRow(); + binding.longColumn("a", 2); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.longColumn("bad", 3)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.longColumn("c", 4); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNamedDesignatedRejection() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.longColumn("value", Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.longColumn("ts", 1)); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; + } + + private static byte wireType(String value) { + return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java new file mode 100644 index 000000000..b6bb8813f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingNativeDecimalTest.java @@ -0,0 +1,572 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Decimal128; +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.Decimal64; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingNativeDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv"; + private static final String HEADER = "# case_id\tsource_type\tsource_scale\tsource_ll_hex\tsource_lh_hex\tsource_hl_hex\tsource_hh_hex\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWireAndPreservesCaller() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingNativeDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + Assert.assertFalse(line.isEmpty()); + String[] f = line.split("\t", -1); + Assert.assertEquals(line, 16, f.length); + Assert.assertTrue("VALUE".equals(f[10]) || "INVALID".equals(f[10])); + assertTargetTag(f[7], Integer.parseInt(f[8])); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", ColumnType.getDecimalType(Integer.parseInt(f[8]), Integer.parseInt(f[9])))); + Object value = decimal(f); + long[] before = limbs(value); + int scale = scale(value); + if ("INVALID".equals(f[10])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> append(binding, value)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(0, buffer.getColumnCount()); + } else { + append(binding, value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireType(f[7])); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Integer.parseInt(f[9]), reader.u8()); + assertExpectedLimbs(reader, f); + Assert.assertEquals(size, reader.position()); + } + Assert.assertEquals(scale, scale(value)); + Assert.assertArrayEquals(before, limbs(value)); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(60, count); + }); + } + + @Test + public void testDecimalTextOverloadUsesEveryExactTargetWidth() throws Exception { + assertMemoryLeak(() -> { + int[] precisions = {2, 4, 9, 18, 38, 76}; + for (int precision : precisions) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", ColumnType.getDecimalType(precision, 1))); + binding.decimalColumn("value", "1.2", new Decimal256()); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, wireTypeForPrecision(precision)); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(12, reader.i64()); + int limbs = precision <= 18 ? 1 : precision <= 38 ? 2 : 4; + for (int i = 1; i < limbs; i++) { + Assert.assertEquals(0, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testDecimalTextOverloadSuppressesDuplicatesAndRollsBackInvalidRows() throws Exception { + assertMemoryLeak(() -> { + int decimal = ColumnType.getDecimalType(5, 1); + Decimal256 scratch = new Decimal256(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("value", decimal), column("only_b", decimal)); + + binding.decimalColumn("value", "1.2", scratch) + .decimalColumn("value", "not-a-decimal", scratch); + buffer.nextRow(); + + binding.decimalColumn("only_b", "2", scratch); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.decimalColumn("value", "not-a-decimal", scratch)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.decimalColumn("value", "NaN", scratch) + .decimalColumn("value", "still-not-a-decimal", scratch); + buffer.nextRow(); + binding.decimalColumn("value", (CharSequence) null, scratch) + .decimalColumn("value", "", scratch); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 3, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x06, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(12, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadValidatesTargetAfterDuplicateBeforeParsing() throws Exception { + assertMemoryLeak(() -> { + Decimal256 scratch = new Decimal256(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.UUID)); + + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", "not-a-decimal", scratch)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.uuidColumn("value", 1, 2) + .decimalColumn("value", "not-a-decimal", scratch); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadMissingInfersDecimal256AtNaturalScale() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + Decimal256 scratch = new Decimal256(); + binding.decimalColumn("value", "NaN", scratch); + buffer.nextRow(); + binding.decimalColumn("value", "12.3400", scratch); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256, 2); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1234, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testDecimalTextOverloadMissingSpecialInfersDecimal256ScaleZeroNull() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.decimalColumn("value", "-Infinity", new Decimal256()); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = unknownTableReader(encoder, size, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.decimalColumn("value", "5.678", new Decimal256()); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = unknownTableReader(encoder, resetSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(3, reset.u8()); + Assert.assertEquals(5678, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testDuplicateRollbackResetOmissionAndRetainedTargetScale() throws Exception { + assertMemoryLeak(() -> { + int decimal = ColumnType.getDecimalType(18, 4); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", decimal), column("bad", ColumnType.getDecimalType(2, 0)), column("c", decimal)); + binding.decimalColumn("a", new Decimal64(15, 1)).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.decimalColumn("a", new Decimal64(2, 0)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.decimalColumn("bad", new Decimal128(0, 1000, 0))); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.decimalColumn("c", new Decimal256(0, 0, 0, 25, 2)); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); + decimalDefinition(reader, "a", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reader, "c", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(4, reader.u8()); + Assert.assertEquals(15000, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(4, reader.u8()); + Assert.assertEquals(2500, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.decimalColumn("c", new Decimal64(3, 0)); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + decimalDefinition(reset, "a", QwpConstants.TYPE_DECIMAL64); + decimalDefinition(reset, "c", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(4, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(4, reset.u8()); + Assert.assertEquals(30000, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingInfersNativeDecimalWidthAndScale() throws Exception { + assertMemoryLeak(() -> { + Object[] values = { + new Decimal64(12, 1), + new Decimal128(3, 4, 2), + new Decimal256(5, 6, 7, 8, 3) + }; + byte[] types = {QwpConstants.TYPE_DECIMAL64, QwpConstants.TYPE_DECIMAL128, QwpConstants.TYPE_DECIMAL256}; + for (int i = 0; i < values.length; i++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + append(binding, values[i]); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = unknownTableReader(encoder, size, types[i]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(scale(values[i]), reader.u8()); + for (long limb : wireLimbs(values[i])) { + Assert.assertEquals(limb, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testMalformedParameterizedAndDesignatedTargetsRemainRejected() throws Exception { + assertMemoryLeak(() -> { + Decimal64 value = new Decimal64(1, 0); + int[] malformed = { + ColumnType.DECIMAL64, + ColumnType.getDecimalType(18, 4) | 0x10000, + ColumnType.getDecimalType(18, 4) | 0x40000000 + }; + for (int type : malformed) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", type)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, + column("value", ColumnType.getDecimalType(18, 4))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + parameterizedColumn("value", ColumnType.getDecimalType(18, 4))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.decimalColumn("value", value)); + } + }); + } + + @Test + public void testLegacyBufferStillChoosesScaleFromFirstValueInEachBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn( + "value", QwpConstants.TYPE_DECIMAL64, true); + column.addDecimal64(new Decimal64(12, 2)); + buffer.nextRow(); + int firstSize = encoder.encode(buffer); + QwpTestWireReader first = legacyTableReader(encoder, firstSize); + Assert.assertEquals(0, first.u8()); + Assert.assertEquals(2, first.u8()); + Assert.assertEquals(12, first.i64()); + Assert.assertEquals(firstSize, first.position()); + + buffer.reset(); + column.addDecimal64(new Decimal64(34, 3)); + buffer.nextRow(); + int secondSize = encoder.encode(buffer); + QwpTestWireReader second = legacyTableReader(encoder, secondSize); + Assert.assertEquals(0, second.u8()); + Assert.assertEquals(3, second.u8()); + Assert.assertEquals(34, second.i64()); + Assert.assertEquals(secondSize, second.position()); + } + }); + } + + @Test + public void testLegacyNullOnlyDecimalUsesWireScaleZeroWithoutLockingNextBatch() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpTableBuffer.ColumnBuffer column = buffer.getOrCreateColumn( + "value", QwpConstants.TYPE_DECIMAL256, true); + column.addDecimal256(Decimal256.NULL_VALUE); + buffer.nextRow(); + + int nullSize = encoder.encode(buffer); + QwpTestWireReader nullOnly = legacyTableReader(encoder, nullSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(1, nullOnly.u8()); + Assert.assertEquals(1, nullOnly.u8()); + Assert.assertEquals(0, nullOnly.u8()); + Assert.assertEquals(nullSize, nullOnly.position()); + + buffer.reset(); + column.addDecimal256(new Decimal256(0, 0, 0, 1234, 4)); + buffer.nextRow(); + int valueSize = encoder.encode(buffer); + QwpTestWireReader value = legacyTableReader(encoder, valueSize, QwpConstants.TYPE_DECIMAL256); + Assert.assertEquals(0, value.u8()); + Assert.assertEquals(4, value.u8()); + Assert.assertEquals(1234, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(0, value.i64()); + Assert.assertEquals(valueSize, value.position()); + } + }); + } + + private static void append(QwpSchemaBinding binding, Object value) { + if (value instanceof Decimal64) { + binding.decimalColumn("value", (Decimal64) value); + } else if (value instanceof Decimal128) { + binding.decimalColumn("value", (Decimal128) value); + } else { + binding.decimalColumn("value", (Decimal256) value); + } + } + + private static void assertExpectedLimbs(QwpTestWireReader reader, String[] f) { + int count = wireLongCount(f[7]); + for (int i = 0; i < count; i++) { + Assert.assertEquals(parseHex(f[11 + i]), reader.i64()); + } + for (int i = count; i < 4; i++) { + Assert.assertEquals("-", f[11 + i]); + } + } + + private static void assertTargetTag(String name, int precision) { + String expected; + if (precision <= 2) { + expected = "DECIMAL8"; + } else if (precision <= 4) { + expected = "DECIMAL16"; + } else if (precision <= 9) { + expected = "DECIMAL32"; + } else if (precision <= 18) { + expected = "DECIMAL64"; + } else if (precision <= 38) { + expected = "DECIMAL128"; + } else { + expected = "DECIMAL256"; + } + Assert.assertEquals(expected, name); + } + + private static Object decimal(String[] f) { + int scale = Integer.parseInt(f[2]); + long ll = parseHex(f[3]); + switch (f[1]) { + case "DECIMAL64": + return new Decimal64(ll, scale); + case "DECIMAL128": + Decimal128 decimal = new Decimal128(); + decimal.of(parseHex(f[4]), ll, scale); + return decimal; + case "DECIMAL256": + return new Decimal256(parseHex(f[6]), parseHex(f[5]), parseHex(f[4]), ll, scale); + default: + throw new AssertionError("unknown source type " + f[1]); + } + } + + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static long[] limbs(Object value) { + if (value instanceof Decimal64) { + return new long[]{((Decimal64) value).getValue()}; + } + if (value instanceof Decimal128) { + return new long[]{((Decimal128) value).getHigh(), ((Decimal128) value).getLow()}; + } + Decimal256 decimal = (Decimal256) value; + return new long[]{decimal.getHh(), decimal.getHl(), decimal.getLh(), decimal.getLl()}; + } + + private static QwpTestWireReader legacyTableReader(QwpWebSocketEncoder encoder, int size) { + return legacyTableReader(encoder, size, QwpConstants.TYPE_DECIMAL64); + } + + private static QwpTestWireReader legacyTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + decimalDefinition(reader, "value", type); + return reader; + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int scale(Object value) { + if (value instanceof Decimal64) { + return ((Decimal64) value).getScale(); + } + if (value instanceof Decimal128) { + return ((Decimal128) value).getScale(); + } + return ((Decimal256) value).getScale(); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static QwpTestWireReader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type) { + return unknownTableReader(encoder, size, type, 1); + } + + private static QwpTestWireReader unknownTableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static byte wireTypeForPrecision(int precision) { + if (precision <= 18) { + return QwpConstants.TYPE_DECIMAL64; + } + if (precision <= 38) { + return QwpConstants.TYPE_DECIMAL128; + } + return QwpConstants.TYPE_DECIMAL256; + } + + private static long[] wireLimbs(Object value) { + if (value instanceof Decimal64) { + return new long[]{((Decimal64) value).getValue()}; + } + if (value instanceof Decimal128) { + Decimal128 decimal = (Decimal128) value; + return new long[]{decimal.getLow(), decimal.getHigh()}; + } + Decimal256 decimal = (Decimal256) value; + return new long[]{decimal.getLl(), decimal.getLh(), decimal.getHl(), decimal.getHh()}; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java new file mode 100644 index 000000000..7caeaed22 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDateTest.java @@ -0,0 +1,185 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringDateTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-date.tsv"; + + @Test + public void testCorpusUsesExactDateWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringDateTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("case_id\tinput_kind\tinput\toutcome\texpected_raw\tfeature", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + CharSequence value = input(fields[1], fields[2]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DATE)); + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + Assert.assertEquals(0, buffer.getRowCount()); + } else { + Assert.assertTrue(line, "VALID".equals(fields[3]) || "NULL".equals(fields[3])); + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", QwpConstants.TYPE_DATE); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(53, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.DATE), column("bad", ColumnType.UUID), column("c", ColumnType.DATE)); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-02"); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-03"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "1969-12-31"); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(13, reader.u8()); + Assert.assertEquals(86_400_000L, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(-86_400_000L, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "1970-01-01 00:00:00.001UTC"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_DATE, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testNativeInferenceParameterizedAndDesignatedBehavior() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "1970-01-01"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.DATE, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "1970-01-01")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("value", ColumnType.DATE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "1970-01-01")); + } + } + + private static CharSequence input(String kind, String value) { + Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); + if ("".equals(value)) { + return null; + } + if ("".equals(value)) { + return ""; + } + if ("TEXT".equals(kind)) { + return value; + } + String[] units = value.split(","); + char[] chars = new char[units.length]; + for (int i = 0; i < units.length; i++) { + chars[i] = (char) Integer.parseInt(units[i], 16); + } + return new String(chars); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java new file mode 100644 index 000000000..ce9ebd77c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringDecimalTest.java @@ -0,0 +1,226 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringDecimalTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-decimal.tsv"; + private static final String HEADER = "# case_id\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql"; + + @Test + public void testCorpusUsesExactTargetDecimalWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringDecimalTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean[] targets = new boolean[6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals(HEADER, lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 11, fields.length); + Vector vector = new Vector(fields); + targets[targetIndex(vector.targetType)] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", ColumnType.getDecimalType(vector.targetPrecision, vector.targetScale))); + if (vector.invalid()) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", vector.input)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(vector.caseId, 0, buffer.getColumnCount()); + } else { + binding.stringColumn("value", vector.input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + Assert.assertEquals(QwpConstants.FLAG_SCHEMA, + Unsafe.getUnsafe().getByte(encoder.getBuffer().getBufferPtr() + + QwpConstants.HEADER_OFFSET_FLAGS) & QwpConstants.FLAG_SCHEMA); + QwpTestWireReader reader = tableReader(encoder, size, wireType(vector.targetType)); + if (vector.isNull()) { + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, 1, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + } else { + Assert.assertEquals(vector.caseId, 0, reader.u8()); + Assert.assertEquals(vector.caseId, vector.targetScale, reader.u8()); + for (int i = 0, n = wireLongCount(vector.targetType); i < n; i++) { + Assert.assertEquals(vector.caseId, vector.expectedLimbs[i], reader.i64()); + } + } + Assert.assertEquals(vector.caseId, size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + vector.caseId + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(55, count); + for (int i = 0; i < targets.length; i++) { + Assert.assertTrue("missing target " + i, targets[i]); + } + }); + } + + @Test + public void testDuplicateNullOmissionAndWholeRowRollback() throws Exception { + assertMemoryLeak(() -> { + int target = ColumnType.getDecimalType(3, 1); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("value", target), column("only_b", target)); + + binding.stringColumn("value", "1.0").stringColumn("value", "invalid"); + buffer.nextRow(); + + binding.stringColumn("only_b", "2.0"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", "100.0")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + binding.stringColumn("value", "-2.5"); + buffer.nextRow(); + binding.stringColumn("value", null).stringColumn("value", "invalid"); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 4, 1); + decimalDefinition(reader, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0c, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(-25, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 1); + decimalDefinition(reset, "value", QwpConstants.TYPE_DECIMAL64); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + private static void decimalDefinition(QwpTestWireReader reader, String name, byte type) { + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + } + + private static long parseHex(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int targetIndex(String target) { + switch (target) { + case "DECIMAL8": return 0; + case "DECIMAL16": return 1; + case "DECIMAL32": return 2; + case "DECIMAL64": return 3; + case "DECIMAL128": return 4; + default: return 5; + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, 1, 1); + decimalDefinition(reader, "value", type); + return reader; + } + + private static byte wireType(String target) { + if ("DECIMAL128".equals(target)) return QwpConstants.TYPE_DECIMAL128; + if ("DECIMAL256".equals(target)) return QwpConstants.TYPE_DECIMAL256; + return QwpConstants.TYPE_DECIMAL64; + } + + private static int wireLongCount(String target) { + if ("DECIMAL128".equals(target)) return 2; + if ("DECIMAL256".equals(target)) return 4; + return 1; + } + + private static final class Vector { + private final String caseId; + private final long[] expectedLimbs = new long[4]; + private final String input; + private final String outcome; + private final int targetPrecision; + private final int targetScale; + private final String targetType; + + private Vector(String[] fields) { + caseId = fields[0]; + input = "".equals(fields[1]) ? null : "".equals(fields[1]) ? "" : fields[1]; + targetType = fields[2]; + targetPrecision = Integer.parseInt(fields[3]); + targetScale = Integer.parseInt(fields[4]); + outcome = fields[5]; + Assert.assertTrue(outcome.equals("VALUE") || outcome.equals("INVALID") || outcome.equals("NULL")); + Assert.assertEquals(expectedTargetType(targetPrecision), targetType); + int limbs = outcome.equals("VALUE") ? wireLongCount(targetType) : 0; + for (int i = 0; i < limbs; i++) expectedLimbs[i] = parseHex(fields[6 + i]); + for (int i = limbs; i < 4; i++) Assert.assertEquals("-", fields[6 + i]); + if (outcome.equals("INVALID")) { + Assert.assertEquals("-", fields[10]); + } else if (outcome.equals("NULL")) { + Assert.assertEquals("NULL", fields[10]); + } else { + Assert.assertNotEquals("-", fields[10]); + Assert.assertNotEquals("NULL", fields[10]); + } + } + + private boolean invalid() { + return outcome.equals("INVALID"); + } + + private boolean isNull() { + return outcome.equals("NULL"); + } + + private static String expectedTargetType(int precision) { + if (precision <= 2) return "DECIMAL8"; + if (precision <= 4) return "DECIMAL16"; + if (precision <= 9) return "DECIMAL32"; + if (precision <= 18) return "DECIMAL64"; + if (precision <= 38) return "DECIMAL128"; + return "DECIMAL256"; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java new file mode 100644 index 000000000..ccc9ac4a3 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringGeoHashTest.java @@ -0,0 +1,280 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringGeoHashTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv"; + private static final String HEADER = "# case_id\tinput\tbits\toutcome\tvalue_hex"; + + @Test + public void testCorpusUsesExactGeoHashWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + boolean headerSeen = false; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse(headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + int bits = Integer.parseInt(fields[2]); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", ColumnType.getGeoHashTypeWithBits(bits)))); + CharSequence value = "".equals(fields[1]) ? null + : "".equals(fields[1]) ? "" : fields[1]; + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + } else if ("VALUE".equals(fields[3]) || "NULL".equals(fields[3])) { + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bits, reader.varint()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(bits, reader.varint()); + long expected = Long.parseUnsignedLong(fields[4], 16); + for (int i = 0; i < (bits + 7) / 8; i++) { + Assert.assertEquals((int) ((expected >>> (8 * i)) & 0xff), reader.u8()); + } + } + Assert.assertEquals(size, reader.position()); + } else { + throw new AssertionError("unknown outcome: " + fields[3]); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue(headerSeen); + Assert.assertEquals(18, count); + }); + } + + @Test + public void testDuplicateBitmapRollbackResetInferenceAndGuards() throws Exception { + assertMemoryLeak(() -> { + int geo8 = ColumnType.getGeoHashTypeWithBits(8); + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("a", geo8), column("bad", ColumnType.LONG), column("c", geo8))); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "zz"); + buffer.nextRow(); + binding.stringColumn("a", ""); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "zz"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "z")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "04"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 5, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + assertGeo(reader, 29, 8, 0xff); + assertGeo(reader, 15, 8, 1); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "04"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reset.u8()); + assertGeo(reset, 0, 8, 1); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(8, reset.varint()); + Assert.assertEquals(resetSize, reset.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "z"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.unsupportedColumn("value", "GEOHASH")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, parameterizedColumn("value", geo8))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "z")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "z")); + } + }); + } + + @Test + public void testMalformedPackedGeoHashTargetsRemainTypedUnsupported() throws Exception { + assertMemoryLeak(() -> { + int geoFlag = 1 << 16; + int[] malformed = { + ColumnType.GEOHASH, + geoFlag | ColumnType.GEOBYTE, + geoFlag | (61 << 8) | ColumnType.GEOLONG, + geoFlag | (8 << 8) | ColumnType.STRING, + ColumnType.getGeoHashTypeWithBits(8) | (1 << 24) + }; + for (int targetType : malformed) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", targetType))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "zz")); + } + } + }); + } + + @Test + public void testAllSixtyBitValuesAndLegacyEncodingRemainUnambiguous() throws Exception { + assertMemoryLeak(() -> { + for (int bits = 1; bits <= 60; bits++) { + int targetType = ColumnType.getGeoHashTypeWithBits(bits); + StringBuilder input = new StringBuilder((bits + 4) / 5); + for (int i = 0; i < (bits + 4) / 5; i++) { + input.append('z'); + } + long max = (1L << bits) - 1; + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(-1, column("value", targetType))); + binding.stringColumn("value", input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + assertGeo(reader, 0, bits, max); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("value", input); + buffer.nextRow(); + binding.stringColumn("value", null); + buffer.nextRow(); + binding.stringColumn("value", ""); + buffer.nextRow(); + size = encoder.encodeSchema(buffer); + reader = tableReader(encoder, size, 3); + assertGeo(reader, 6, bits, max); + Assert.assertEquals(size, reader.position()); + } + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + buffer.getOrCreateColumn("value", QwpConstants.TYPE_GEOHASH, true) + .addGeoHash(0x0fffffffffffffffL, 60); + buffer.nextRow(); + int size = encoder.encode(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(60, reader.varint()); + Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + buffer.getOrCreateColumn("value", QwpConstants.TYPE_GEOHASH, false) + .addGeoHash(0x0fffffffffffffffL, 60); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer, 1, 1); + QwpTestWireReader reader = tableReader(encoder, size, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(60, reader.varint()); + Assert.assertEquals(0x0fffffffffffffffL, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + private static void assertGeo(QwpTestWireReader reader, int bitmap, int bits, long value) { + Assert.assertEquals(bitmap < 0 ? 0 : 1, reader.u8()); + if (bitmap >= 0) { + Assert.assertEquals(bitmap, reader.u8()); + } + Assert.assertEquals(bits, reader.varint()); + for (int i = 0; i < (bits + 7) / 8; i++) { + Assert.assertEquals((int) ((value >>> (8 * i)) & 0xff), reader.u8()); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java new file mode 100644 index 000000000..a1023ca4a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringLong256Test.java @@ -0,0 +1,221 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringLong256Test { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv"; + private static final String HEADER = "# case_id\tinput\toutcome\tl0_hex\tl1_hex\tl2_hex\tl3_hex"; + + @Test + public void testCorpusUsesExactLong256Wire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringLong256Test.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + boolean headerSeen = false; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals(HEADER, line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.LONG256)); + if ("VALUE".equals(fields[2])) { + binding.stringColumn("value", fields[1]); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + Assert.assertEquals(0, reader.u8()); + for (int i = 3; i < 7; i++) { + Assert.assertEquals(parseHexLong(fields[i]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } else if ("NULL".equals(fields[2])) { + Assert.assertEquals("", fields[1]); + binding.stringColumn("value", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } else if ("INVALID".equals(fields[2])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", fields[1])); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + Assert.assertEquals(0, buffer.getColumnDefs().length); + } else { + throw new AssertionError("unknown outcome: " + fields[2]); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(23, count); + }); + } + + @Test + public void testDuplicateNullOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.LONG256), column("bad", ColumnType.LONG), column("c", ColumnType.LONG256)); + binding.stringColumn("a", null).binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("a", "0x01").stringColumn("a", "invalid duplicate"); + buffer.nextRow(); + buffer.nextRow(); + binding.stringColumn("a", "0x02"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "0x03")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "0x04"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(4, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + assertLong256Values(reader, 13, 1); + assertLong256Values(reader, 7, 4); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("a", "0x05"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(5, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(0, reset.i64()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testMissingStringInferenceRemainsVarchar() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "0x00"); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, buffer.getColumnDefs()[0].getTypeCode()); + } + } + + @Test + public void testParameterizedDesignatedAndOtherTargetsRemainUnsupported() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + parameterizedColumn("value", ColumnType.LONG256), column("other", ColumnType.UUID)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("value", "0x00")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("other", "0x00")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "0x00")); + } + } + + private static void assertLong256Values(QwpTestWireReader reader, int bitmap, long value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(value, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + Assert.assertEquals(0, reader.i64()); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + return tableReader(encoder, size, rows, "value"); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(column, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_LONG256, reader.u8()); + return reader; + } + + private static long parseHexLong(String value) { + return Long.parseUnsignedLong(value, 16); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java new file mode 100644 index 000000000..754464584 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringNumericTest.java @@ -0,0 +1,292 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTablePrefix; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringNumericTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-numeric.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringNumericTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') continue; + String[] f = line.split("\\t", -1); + Assert.assertEquals(line, 7, f.length); + Assert.assertTrue(line, "VALID".equals(f[6]) || "INVALID".equals(f[6])); + CharSequence value = "".equals(f[1]) ? null : utf16(f[1]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", targetType(f[2]))); + if ("INVALID".equals(f[6])) { + Assert.assertEquals(f[0], "", f[4]); + Assert.assertEquals(f[0], "", f[5]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("value", value)); + } else { + binding.stringColumn("value", value); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType(f[3]), 1, 1); + if ("".equals(f[4])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + byte[] expected = littleEndianHex(f[4]); + Assert.assertArrayEquals(f[0], expected, reader.bytes(expected.length)); + } + Assert.assertEquals(f[0], encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError | RuntimeException e) { + throw new AssertionError("case_id=" + f[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(429, count); + }); + } + + @Test + public void testDuplicatePrecedesInvalidParseAndUnsupportedConversion() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("n", ColumnType.INT), column("uuid", ColumnType.UUID)); + binding.stringColumn("n", "42").stringColumn("n", "invalid"); + binding.uuidColumn("uuid", 1, 2).stringColumn("uuid", "not-a-uuid"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 1, 2); + Assert.assertEquals("n", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals("uuid", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(42, reader.i32()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.TIMESTAMP_MICRO)); + binding.timestampColumn("value", 123, ChronoUnit.MICROS) + .stringColumn("value", "not-a-timestamp-conversion"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_TIMESTAMP, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(123, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testFailureRollbackRemovesFailedOnlyColumnAndPreservesRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.INT), column("only_b", ColumnType.LONG), + column("bad", ColumnType.SHORT), column("c", ColumnType.DOUBLE)); + binding.stringColumn("a", "1"); + buffer.nextRow(); + binding.stringColumn("only_b", "2"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> binding.stringColumn("bad", "+3")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "-0.0"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_INT, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_DOUBLE, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x8000000000000000L, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testResetClearAndMutableInput() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.LONG)); + StringBuilder mutable = new StringBuilder("7"); + binding.stringColumn("value", mutable); + mutable.replace(0, 1, "9"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + + buffer.reset(); + binding.stringColumn("value", "8"); + buffer.nextRow(); + reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_LONG, 1, 1); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(8, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + buffer.clear(); + assertIllegalState(() -> binding.stringColumn("value", "1")); + } + }); + } + + @Test + public void testNullAndParsedSentinelsHaveDistinctWirePresence() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.INT)); + binding.stringColumn("value", "-2147483648"); + buffer.nextRow(); + binding.stringColumn("value", null); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), QwpConstants.TYPE_INT, 2, 1); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(Integer.MIN_VALUE, reader.i32()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("bool", ColumnType.BOOLEAN), column("date", ColumnType.DATE, new byte[]{1}), + column("future", ColumnType.INT, new byte[]{1}), + column("flagged", ColumnType.LONG | 0x10000)); + assertSchemaError(LineSenderSchemaException.Reason.INVALID_VALUE, "bool", "BOOLEAN", () -> binding.stringColumn("bool", "2")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "date", "DATE", () -> binding.stringColumn("date", "1")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "future", "INT", () -> binding.stringColumn("future", "1")); + rollback(buffer); + assertSchemaError(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "flagged", "unknown", () -> binding.stringColumn("flagged", "1")); + rollback(buffer); + binding.stringColumn("missing", "1"); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("d", ColumnType.DOUBLE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> binding.stringColumn("d", null)); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte type, int rows, int columns) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTablePrefix(reader, rows, columns); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError(value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError(value); + } + } + + private static String utf16(String hex) { + Assert.assertEquals(0, hex.length() & 3); + char[] chars = new char[hex.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(hex.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static byte[] littleEndianHex(String hex) { + Assert.assertEquals(0, hex.length() & 1); + byte[] bytes = new byte[hex.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(hex.substring((bytes.length - 1 - i) * 2, (bytes.length - i) * 2), 16); + } + return bytes; + } + + private static void rollback(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static void assertIllegalState(Runnable runnable) { + Assert.assertThrows(IllegalStateException.class, runnable::run); + } + + private static void assertSchemaError( + LineSenderSchemaException.Reason reason, + String column, + String target, + Runnable runnable + ) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, runnable::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("table=t")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("column=" + column)); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("inputType=STRING")); + Assert.assertTrue(error.getMessage(), error.getMessage().contains("targetType=" + target)); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java new file mode 100644 index 000000000..701fbea43 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingStringTimestampTest.java @@ -0,0 +1,213 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingStringTimestampTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv"; + + @Test + public void testCorpusUsesExactTargetUnitWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingStringTimestampTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("case_id\ttarget\tinput_kind\tinput\toutcome\texpected_raw\tfeature", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertTrue(line, "VALID".equals(fields[4]) || "NULL".equals(fields[4]) || "INVALID".equals(fields[4])); + int target = targetType(fields[1]); + byte wireType = wireType(fields[1]); + CharSequence value = input(fields[2], fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + if ("INVALID".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("value", value)); + Assert.assertEquals(0, buffer.getRowCount()); + count++; + continue; + } + binding.stringColumn("value", value); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, "value", wireType); + if ("NULL".equals(fields[4])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(110, count); + }); + } + + @Test + public void testNullDuplicateOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int target : new int[]{ColumnType.TIMESTAMP_MICRO, ColumnType.TIMESTAMP_NANO}) { + byte wireType = target == ColumnType.TIMESTAMP_MICRO + ? QwpConstants.TYPE_TIMESTAMP + : QwpConstants.TYPE_TIMESTAMP_NANOS; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + binding.stringColumn("value", null).binaryColumn("value", new byte[]{1}); + buffer.nextRow(); + binding.stringColumn("value", "1970-01-01T00:00:00.000001Z"); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, "value", wireType); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? 1 : 1000, reader.i64()); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.stringColumn("value", "1969-12-31T23:59:59.999999Z"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableReader(encoder, resetSize, 1, "value", wireType); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(target == ColumnType.TIMESTAMP_MICRO ? -1 : -1000, reset.i64()); + Assert.assertEquals(resetSize, reset.position()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", ColumnType.TIMESTAMP_MICRO), + column("bad", ColumnType.UUID), + column("c", ColumnType.TIMESTAMP_NANO)); + binding.stringColumn("a", "1969-12-31T23:59:59.999999Z"); + buffer.nextRow(); + binding.stringColumn("a", "1970-01-01T00:00:00.000002Z"); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.stringColumn("bad", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.stringColumn("c", "1970-01-01T00:00:00.000004Z"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 2, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4000, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNamedDesignatedRejection() { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.stringColumn("value", "1970-01-01T00:00:00Z"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(20, reader.i32()); + Assert.assertEquals("1970-01-01T00:00:00Z", reader.ascii(20)); + Assert.assertEquals(size, reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("ts", "1970-01-01T00:00:00Z")); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String name, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals(name, reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } + + private static int targetType(String value) { + Assert.assertTrue(value, "TIMESTAMP".equals(value) || "TIMESTAMP_NANOS".equals(value)); + return "TIMESTAMP".equals(value) ? ColumnType.TIMESTAMP_MICRO : ColumnType.TIMESTAMP_NANO; + } + + private static CharSequence input(String kind, String value) { + Assert.assertTrue(kind, "TEXT".equals(kind) || "UTF16_HEX".equals(kind)); + if ("".equals(value)) { + return null; + } + if ("TEXT".equals(kind)) { + return value; + } + String[] units = value.split(","); + char[] chars = new char[units.length]; + for (int i = 0; i < units.length; i++) { + chars[i] = (char) Integer.parseInt(units[i], 16); + } + return new String(chars); + } + + private static byte wireType(String value) { + return "TIMESTAMP".equals(value) ? QwpConstants.TYPE_TIMESTAMP : QwpConstants.TYPE_TIMESTAMP_NANOS; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java new file mode 100644 index 000000000..80c16439e --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTest.java @@ -0,0 +1,2234 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpColumnDef; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.UUID; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.schemaResult; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingTest { + @Test + public void testTimestampInputConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + int count = 0; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + try { + int targetType = timestampColumnType(fields[4]); + byte wireType = timestampWireType(fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + Runnable write; + if ("LONG".equals(fields[1])) { + long input = Long.parseLong(fields[2]); + ChronoUnit unit = ChronoUnit.valueOf(fields[3]); + write = () -> rows.timestampColumn("ts", input, unit); + } else if ("INSTANT".equals(fields[1])) { + Instant input = Instant.parse(fields[2]); + Assert.assertEquals("-", fields[3]); + write = () -> rows.timestampColumn("ts", input); + } else { + throw new AssertionError("unknown input_kind: " + fields[1]); + } + if ("".equals(fields[5])) { + Assert.assertEquals("", fields[6]); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, write); + continue; + } + write.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.u8()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[5]), reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(102, count); + } + } + + @Test + public void testTimestampOverloadValidationAndDuplicateSuppression() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("ts", Instant.ofEpochSecond(0, 123)) + .timestampColumn("ts", (Instant) null) + .timestampColumn("ts", Long.MAX_VALUE, ChronoUnit.WEEKS); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, + QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(123, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", (Instant) null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.WEEKS)); + } + for (int targetType : new int[]{ColumnType.LONG, ColumnType.TIMESTAMP_NANO | 0x10000}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", (Instant) null)); + } + } + }); + } + + @Test + public void testTimestampInstantFailureRollsBackOnlyCurrentRowColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("late", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("micro", Instant.ofEpochSecond(1)); + buffer.nextRow(); + rows.timestampColumn("micro", Instant.ofEpochSecond(2)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("late", Instant.MAX)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.timestampColumn("micro", Instant.ofEpochSecond(3)) + .timestampColumn("late", Instant.ofEpochSecond(0, 4)); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1_000_000, reader.i64()); + Assert.assertEquals(3_000_000, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testTimestampUnitConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/timestamp-units.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + int count = 0; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try { + long input = Long.parseLong(fields[1]); + ChronoUnit unit = ChronoUnit.valueOf(fields[2]); + int targetType = timestampColumnType(fields[3]); + byte wireType = timestampWireType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + if ("".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", input, unit)); + continue; + } + rows.timestampColumn("ts", input, unit); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + Assert.assertEquals("supplied timestamp must be bitmap-present", 0, reader.u8()); + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(48, count); + } + } + + @Test + public void testTimestampValidationDuplicateAndRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("nano", ColumnType.TIMESTAMP_NANO)); + rows.timestampColumn("micro", 7, ChronoUnit.MICROS) + .timestampColumn("micro", Long.MAX_VALUE, null) + .timestampColumn("micro", Long.MAX_VALUE, ChronoUnit.SECONDS); + buffer.nextRow(); + + rows.timestampColumn("micro", 8, ChronoUnit.MICROS); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("nano", Long.MAX_VALUE, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + + rows.timestampColumn("micro", 9, ChronoUnit.MICROS) + .timestampColumn("nano", 10, ChronoUnit.NANOS); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(9, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(10, reader.i64()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testTimestampNullUnsupportedAndBindingLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("micro", ColumnType.TIMESTAMP_MICRO), + column("nano", ColumnType.TIMESTAMP_NANO)); + rows.stringColumn("micro", null).stringColumn("nano", null); + buffer.nextRow(); + rows.stringColumn("micro", null); + buffer.nextRow(); + QwpTestWireReader nulls = tableReader(encoder, encoder.encodeSchema(buffer), 2, + QwpConstants.TYPE_TIMESTAMP, QwpConstants.TYPE_TIMESTAMP_NANOS); + Assert.assertEquals(1, nulls.u8()); + Assert.assertEquals(3, nulls.u8()); + Assert.assertEquals(0, nulls.u8()); + Assert.assertEquals(1, nulls.u8()); + Assert.assertEquals(3, nulls.u8()); + Assert.assertEquals(0, nulls.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), nulls.position()); + + buffer.reset(); + rows.timestampColumn("nano", 11, ChronoUnit.NANOS); + buffer.nextRow(); + buffer.clear(); + assertIllegalState(() -> rows.timestampColumn("nano", 12, ChronoUnit.NANOS)); + assertIllegalState(() -> rows.timestampColumn("nano", Instant.EPOCH)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("x", ColumnType.LONG)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("x", Long.MIN_VALUE, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.timestampColumn("missing", 1, ChronoUnit.MICROS); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.timestampColumn("ts", 1, null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.WEEKS)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", ColumnType.TIMESTAMP_MICRO, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + for (int targetType : new int[]{ColumnType.TIMESTAMP_MICRO | 0x10000, ColumnType.TIMESTAMP_NANO | 0x10000}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ts", targetType)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", 1, ChronoUnit.MICROS)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.timestampColumn("ts", Instant.EPOCH)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.stringColumn("ts", null)); + } + }); + } + @Test + public void testClassifiesSchemaLessResponses() throws Exception { + assertCtorReason(QwpSchemaProtocol.RESULT_DENIED, LineSenderSchemaException.Reason.ACCESS_DENIED, "access denied"); + assertCtorReason(QwpSchemaProtocol.RESULT_UNAVAILABLE, LineSenderSchemaException.Reason.SCHEMA_UNAVAILABLE, "temporarily unavailable"); + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)).longColumn("x", 1); + } + assertCtorReason(QwpSchemaProtocol.RESULT_TOO_LARGE, LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, "response limit"); + } + + @Test + public void testInfersNativeTargetsForMissingSchemaAndColumns() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.longColumn("l", 1) + .stringColumn("s", "x") + .symbol("sym", "y") + .boolColumn("b", true) + .floatColumn("f", 1) + .doubleColumn("d", 2) + .uuidColumn("u", 3, 4) + .binaryColumn("bin", new byte[]{5}) + .timestampColumn("tm", 6, ChronoUnit.MICROS) + .timestampColumn("tn", 7, ChronoUnit.NANOS) + .timestampColumn("ti", Instant.EPOCH); + buffer.nextRow(); + byte[] expected = { + QwpConstants.TYPE_LONG, QwpConstants.TYPE_VARCHAR, QwpConstants.TYPE_SYMBOL, + QwpConstants.TYPE_BOOLEAN, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE, + QwpConstants.TYPE_UUID, QwpConstants.TYPE_BINARY, QwpConstants.TYPE_TIMESTAMP, + QwpConstants.TYPE_TIMESTAMP_NANOS, QwpConstants.TYPE_TIMESTAMP + }; + QwpColumnDef[] definitions = buffer.getColumnDefs(); + Assert.assertEquals(expected.length, definitions.length); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals("column " + i, expected[i], definitions[i].getTypeCode()); + } + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.uuidColumn("u", 0x0102030405060708L, 0x1112131415161718L) + .floatColumn("f", Float.intBitsToFloat(0x3fc00000)) + .stringColumn("s", "quest"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, + QwpConstants.TYPE_UUID, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x0102030405060708L, reader.i64()); + Assert.assertEquals(0x1112131415161718L, reader.i64()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0x3fc00000, reader.i32()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(5, reader.i32()); + Assert.assertEquals("quest", reader.ascii(5)); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("known", ColumnType.LONG)); + rows.longColumn("missing", 1) + .unsupportedColumn("missing", "INT") + .uuidColumn("missing", 2, 3); + buffer.nextRow(); + LineSenderSchemaException conflict = Assert.assertThrows( + LineSenderSchemaException.class, + () -> rows.uuidColumn("missing", 4, 5) + ); + Assert.assertEquals(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, conflict.getReason()); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.longColumn("a", 1); + buffer.nextRow(); + try { + rows.stringColumn("failed_only", new CharSequence() { + @Override + public char charAt(int index) { + if (index == 1) { + throw new IllegalStateException("fixture failure"); + } + return 'x'; + } + + @Override + public int length() { + return 2; + } + + @Override + public CharSequence subSequence(int start, int end) { + throw new UnsupportedOperationException(); + } + }); + Assert.fail("expected fixture failure"); + } catch (IllegalStateException e) { + Assert.assertEquals("fixture failure", e.getMessage()); + } + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("a", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer micros = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(micros, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.designatedTimestamp(1, ChronoUnit.MICROS); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP, micros.getColumnDefs()[0].getTypeCode()); + micros.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.designatedTimestamp(2, ChronoUnit.NANOS)); + micros.cancelCurrentRow(); + micros.rollbackUncommittedColumns(); + } + try (QwpTableBuffer nanos = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(nanos, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.designatedTimestamp(1, ChronoUnit.NANOS); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, nanos.getColumnDefs()[0].getTypeCode()); + } + }); + } + + @Test + public void testDuplicateUsesFirstValueAcrossInputSetters() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.uuidColumn("u", 1, 2).longColumn("u", 3); + rows.longColumn("l", 4).stringColumn("l", "not-a-long"); + buffer.nextRow(); + rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef") + .stringColumn("u", "malformed"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testEmptyAndIncompleteEncodeContracts() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("l", ColumnType.LONG)); + Assert.assertEquals(0, encoder.encodeSchema(buffer)); + rows.longColumn("l", 1); + assertIllegalState(() -> encoder.encodeSchema(buffer)); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 1); + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + buffer.reset(); + Assert.assertEquals(0, encoder.encodeSchema(buffer)); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + }); + } + + @Test + public void testLifecycleAndNameValidation() throws Exception { + try (QwpTableBuffer invalid = new QwpTableBuffer("bad\nname")) { + assertIllegalArgument(() -> new QwpSchemaBinding(invalid, known(column("x", ColumnType.LONG)))); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("x", ColumnType.LONG)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("bad\nname", 1)); + rows.longColumn("missing", 1); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding stale = rows(buffer, column("x", ColumnType.LONG)); + buffer.clear(); + assertIllegalState(() -> stale.longColumn("x", 1)); + assertIllegalState(() -> stale.timestampColumn("x", Instant.EPOCH)); + } + } + + @Test + public void testSmallIntegerNumericConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + try { + long input = Long.parseLong(fields[2]); + int targetType = numericColumnType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", targetType)); + Runnable append = () -> appendSmallInteger(rows, fields[1], input, "n"); + if ("".equals(fields[4])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + } else { + append.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, + numericWireType(fields[3])); + if ("".equals(fields[4])) { + assertNumericNull(reader, fields[3]); + } else { + Assert.assertEquals(0, reader.u8()); + assertNumericValue(reader, fields[3], fields[4]); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(64, count); + } + + @Test + public void testSmallIntegerTextConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + try { + long input = Long.parseLong(fields[2]); + byte wireType = textWireType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", textColumnType(fields[3]))); + appendSmallInteger(rows, fields[1], input, "value"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + if ("".equals(fields[4])) { + Assert.assertEquals("", fields[5]); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("row zero must be NULL", 1, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + } + } else { + byte[] expected = hexBytes(fields[4]); + Assert.assertEquals(fields[5], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals("value must be bitmap-present", 0, reader.u8()); + if (wireType == QwpConstants.TYPE_SYMBOL) { + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(fields[5], reader.string()); + Assert.assertEquals(0, reader.varint()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + for (byte value : expected) { + Assert.assertEquals(value & 0xff, reader.u8()); + } + } + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(30, count); + } + + @Test + public void testSmallIntegerDecimalConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv"); + Assert.assertNotNull(stream); + boolean[][] sourceTargets = new boolean[3][6]; + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals( + "# case_id\tinput_type\tinput\ttarget_type\ttarget_precision\ttarget_scale\toutcome" + + "\texpected_ll_hex\texpected_lh_hex\texpected_hl_hex\texpected_hh_hex\texpected_sql", + lines.readLine() + ); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 12, fields.length); + try { + long input = Long.parseLong(fields[2]); + int precision = Integer.parseInt(fields[4]); + int scale = Integer.parseInt(fields[5]); + Assert.assertEquals(decimalStorageType(precision), fields[3]); + sourceTargets[smallIntegerSourceIndex(fields[1])][decimalStorageIndex(fields[3])] = true; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows( + buffer, + column("value", ColumnType.getDecimalType(precision, scale)) + ); + Runnable append = () -> appendSmallInteger(rows, fields[1], input, "value"); + if ("INVALID".equals(fields[6])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + rollbackCurrentRow(buffer); + } else { + append.run(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader( + encoder, + size, + 1, + decimalWireType(fields[3]) + ); + if ("NULL".equals(fields[6])) { + Assert.assertEquals("NULL", fields[11]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + } else { + Assert.assertEquals("VALUE", fields[6]); + Assert.assertEquals(0, reader.u8()); + } + Assert.assertEquals(scale, reader.u8()); + if ("VALUE".equals(fields[6])) { + int limbs = decimalWireLongCount(fields[3]); + for (int i = 0; i < limbs; i++) { + Assert.assertEquals(parseHexLong(fields[7 + i]), reader.i64()); + } + for (int i = limbs; i < 4; i++) { + Assert.assertEquals("-", fields[7 + i]); + } + } + Assert.assertEquals(size, reader.position()); + } + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(30, count); + for (int source = 0; source < sourceTargets.length; source++) { + for (int target = 0; target < sourceTargets[source].length; target++) { + Assert.assertTrue("missing source/target coverage " + source + '/' + target, + sourceTargets[source][target]); + } + } + } + + @Test + public void testIntegerTemporalConformanceCorpusUsesTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + try { + long input = Long.parseLong(fields[2]); + int targetType = temporalColumnType(fields[3]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + appendIntegerTemporal(rows, fields[1], input, "value"); + buffer.nextRow(); + QwpTestWireReader reader = tableReader( + encoder, + encoder.encodeSchema(buffer), + 1, + temporalWireType(fields[3]) + ); + if ("".equals(fields[4])) { + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("row zero must be NULL", 1, reader.u8()); + } else { + Assert.assertEquals("value must be bitmap-present", 0, reader.u8()); + } + if (!"DATE".equals(fields[3])) { + Assert.assertEquals("short timestamp columns use raw encoding", 0, reader.u8()); + } + if (!"".equals(fields[4])) { + Assert.assertEquals(Long.parseLong(fields[4]), reader.i64()); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(24, count); + } + + @Test + public void testSmallIntegerDuplicateFirstAndFailureRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("a", ColumnType.LONG), + column("only_b", ColumnType.SHORT), + column("bad", ColumnType.BYTE), + column("c", ColumnType.DOUBLE)); + rows.byteColumn("a", (byte) -1).intColumn("a", Integer.MIN_VALUE); + buffer.nextRow(); + + rows.shortColumn("only_b", (short) 5); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.shortColumn("bad", (short) 128)); + rollbackCurrentRow(buffer); + + rows.intColumn("c", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, + QwpConstants.TYPE_LONG, QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(-1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(Double.doubleToRawLongBits(3), reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testIntMinIsNullWithCompanionMissingRowsAcrossNumericTargets() throws Exception { + assertMemoryLeak(() -> { + for (String target : new String[]{"BYTE", "SHORT", "INT", "LONG", "FLOAT", "DOUBLE"}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", numericColumnType(target))); + rows.intColumn("n", Integer.MIN_VALUE); + buffer.nextRow(); + buffer.nextRow(); + rows.intColumn("n", 7); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, numericWireType(target)); + Assert.assertEquals(target, 1, reader.u8()); + Assert.assertEquals(target, 3, reader.u8()); + if ("FLOAT".equals(target)) { + assertNumericValue(reader, target, "0x40e00000"); + } else if ("DOUBLE".equals(target)) { + assertNumericValue(reader, target, "0x401c000000000000"); + } else { + assertNumericValue(reader, target, "7"); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testIntToIpv4UsesTargetNativeWireAndNulls() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ip", ColumnType.IPv4)); + rows.intColumn("ip", Integer.MIN_VALUE); + buffer.nextRow(); + rows.intColumn("ip", 0); + buffer.nextRow(); + rows.intColumn("ip", 1); + buffer.nextRow(); + rows.intColumn("ip", Integer.MIN_VALUE + 1); + buffer.nextRow(); + rows.intColumn("ip", Integer.MAX_VALUE); + buffer.nextRow(); + rows.intColumn("ip", -1); + buffer.nextRow(); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 7, QwpConstants.TYPE_IPv4); + Assert.assertEquals("NULL bitmap must be present", 1, reader.u8()); + Assert.assertEquals("INT null, IPv4 zero and omission must be NULL", 0x43, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(Integer.MIN_VALUE + 1, reader.i32()); + Assert.assertEquals(Integer.MAX_VALUE, reader.i32()); + Assert.assertEquals(-1, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testSmallIntegerMissingSchemaUsesNativeWireTypes() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.byteColumn("b", (byte) -1).shortColumn("s", (short) 2).intColumn("i", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, + QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0xff, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(2, reader.u16()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(3, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testSmallIntegerUnsupportedParameterizedDesignatedAndStaleTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("uuid", ColumnType.UUID), + column("ip", ColumnType.IPv4), + column("future", ColumnType.INT, new byte[]{1}), + column("future_date", ColumnType.DATE, new byte[]{1}), + column("future_text", ColumnType.VARCHAR, new byte[]{1}), + column("future_decimal", ColumnType.getDecimalType(18, 2), new byte[]{1}), + column("future_ipv4", ColumnType.IPv4, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("uuid", (byte) 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("ip", (byte) 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("ip", (short) 2)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("future", (short) 2)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.byteColumn("future_date", (byte) 3)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("future_text", 4)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.shortColumn("future_decimal", (short) 5)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("future_ipv4", 6)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("uuid", Integer.MIN_VALUE)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("designated", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.intColumn("designated", 1)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding stale = rows(buffer, column("n", ColumnType.LONG)); + buffer.clear(); + assertIllegalState(() -> stale.byteColumn("n", (byte) 1)); + assertIllegalState(() -> stale.shortColumn("n", (short) 2)); + assertIllegalState(() -> stale.intColumn("n", 3)); + } + } + + @Test + public void testLongNumericConformanceCorpus() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/long-to-numeric.tsv"); + Assert.assertNotNull(stream); + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + int count = 0; + while ((line = lines.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 4, fields.length); + try { + long input = Long.parseLong(fields[1]); + int targetType = numericColumnType(fields[2]); + byte wireType = numericWireType(fields[2]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("n", targetType)); + if ("".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.longColumn("n", input)); + continue; + } + rows.longColumn("n", input); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); + if ("".equals(fields[3])) { + if (input == Long.MIN_VALUE) { + assertNumericNull(reader, fields[2]); + } else { + Assert.assertEquals("INT sentinel is carried as a non-null wire value", 0, reader.u8()); + Assert.assertEquals(Integer.MIN_VALUE, reader.i32()); + } + } else { + Assert.assertEquals(0, reader.u8()); + assertNumericValue(reader, fields[2], fields[3]); + } + Assert.assertEquals(size, reader.position()); + } + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + // An emptied or unparsed corpus must not pass vacuously. + Assert.assertEquals(47, count); + } + } + + @Test + public void testLongNumericDuplicateFirstAndFailureRollback() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("b", ColumnType.BYTE), column("i", ColumnType.INT)); + rows.longColumn("b", 7).longColumn("b", 128).stringColumn("b", "unsupported"); + rows.longColumn("i", 1); + buffer.nextRow(); + rows.longColumn("i", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("b", 128)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("b", 3).longColumn("i", 4); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_INT); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(3, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(4, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLongRangeDiagnosticIsReadableAndDoesNotEchoValue() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("b", ColumnType.BYTE)); + try { + rows.longColumn("b", 9_876_543_210L); + Assert.fail("expected range failure"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(LineSenderSchemaException.Reason.INVALID_VALUE, e.getReason()); + Assert.assertFalse(e.isRetryable()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("table=t")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("column=b")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("inputType=LONG")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("targetType=BYTE(" + ColumnType.BYTE + ")")); + Assert.assertTrue(e.getMessage(), e.getMessage().contains("outside target type range")); + Assert.assertFalse(e.getMessage(), e.getMessage().contains("9876543210")); + } + } + } + + @Test + public void testLongNullRejectsUuidUnlessDuplicate() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.longColumn("u", Long.MIN_VALUE)); + rows.uuidColumn("u", 1, 2).longColumn("u", Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLongNumericDuplicateFirstAcrossEveryTarget() throws Exception { + assertMemoryLeak(() -> { + byte[][] schema = { + column("b", ColumnType.BYTE), column("s", ColumnType.SHORT), + column("i", ColumnType.INT), column("l", ColumnType.LONG), + column("f", ColumnType.FLOAT), column("d", ColumnType.DOUBLE)}; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, schema); + String[] names = {"b", "s", "i", "l", "f", "d"}; + for (String name : names) { + rows.longColumn(name, 7).stringColumn(name, "not-numeric").uuidColumn(name, 1, 2); + } + rows.longColumn("b", 128).longColumn("s", 32768).longColumn("i", 2147483648L); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.u8()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.u16()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.i32()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Float.floatToRawIntBits(7), reader.i32()); + Assert.assertEquals(0, reader.u8()); Assert.assertEquals(Double.doubleToRawLongBits(7), reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNumericFailureRemovesColumnsIntroducedOnlyInFailedRow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("a", ColumnType.LONG), column("only_b", ColumnType.BYTE), + column("bad", ColumnType.SHORT)); + rows.longColumn("a", 1); + buffer.nextRow(); + rows.longColumn("only_b", 5); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.longColumn("bad", 32768)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("a", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNumericExtensionRejectedOnlyWhenUsed() throws Exception { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("future_float", ColumnType.FLOAT, new byte[]{1}), column("plain", ColumnType.BYTE)); + rows.longColumn("plain", 1); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.longColumn("future_float", 2)); + } + } + + @Test + public void testNullStringSupportsAllNumericTargets() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("b", ColumnType.BYTE), column("s", ColumnType.SHORT), + column("i", ColumnType.INT), column("l", ColumnType.LONG), + column("f", ColumnType.FLOAT), column("d", ColumnType.DOUBLE)); + rows.stringColumn("b", null).stringColumn("s", null).stringColumn("i", null) + .stringColumn("l", null).stringColumn("f", null).stringColumn("d", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_BYTE, QwpConstants.TYPE_SHORT, + QwpConstants.TYPE_INT, QwpConstants.TYPE_LONG, QwpConstants.TYPE_FLOAT, QwpConstants.TYPE_DOUBLE); + assertNumericNull(reader, "BYTE"); + assertNumericNull(reader, "SHORT"); + assertNumericNull(reader, "INT"); + assertNumericNull(reader, "LONG"); + assertNumericNull(reader, "FLOAT"); + assertNumericNull(reader, "DOUBLE"); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testNullStringSupportsLongAndUuidTargets() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.stringColumn("u", null).stringColumn("l", null); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_UUID, QwpConstants.TYPE_LONG); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testParameterExtensionRejectedOnlyWhenColumnUsed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("future", ColumnType.LONG, new byte[]{1}), column("plain", ColumnType.LONG)); + rows.longColumn("plain", 1); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 1); + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.longColumn("future", 2)); + } + }); + } + + @Test + public void testUnknownFullTypeFlagsRejectedOnlyWhenUsed() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("future", ColumnType.LONG | 0x10000), column("plain", ColumnType.LONG)); + rows.longColumn("plain", 7); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 7); + buffer.reset(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> rows.longColumn("future", 2)); + } + }); + } + + @Test + public void testCancelAndResetRemoveOnlyIntendedRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("l", ColumnType.LONG)); + rows.longColumn("l", 1); + buffer.nextRow(); + rows.longColumn("l", 2); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.longColumn("l", 3); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 1, 3); + buffer.reset(); + rows.longColumn("l", 4); + buffer.nextRow(); + size = encoder.encodeSchema(buffer); + assertLongFrame(encoder, size, 4); + } + }); + } + + @Test + public void testOwnerRollbackAfterSetterFailurePreservesCompletedRows() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + rows.longColumn("l", 1); + buffer.nextRow(); + rows.longColumn("l", 2); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.stringColumn("u", "not-a-uuid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + rows.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_LONG, QwpConstants.TYPE_UUID); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(2, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(0x0123456789abcdefL, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testAttachmentLifecycleAndStaleBindingGuards() throws Exception { + assertMemoryLeak(() -> { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaResponse schema = known(column("u", ColumnType.UUID), column("l", ColumnType.LONG)); + QwpSchemaBinding old = new QwpSchemaBinding(buffer, schema); + assertIllegalState(() -> new QwpSchemaBinding(buffer, schema)); + buffer.reset(); + old.longColumn("l", 1); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + buffer.clear(); + + QwpSchemaBinding current = new QwpSchemaBinding(buffer, schema); + current.longColumn("l", 7); + assertIllegalState(() -> old.longColumn("l", 8)); + assertIllegalState(() -> old.stringColumn("u", "01234567-89ab-cdef-0123-456789abcdef")); + assertIllegalState(() -> old.uuidColumn("u", 1, 2)); + buffer.nextRow(); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(7, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testBindingRequiresEmptyLayout() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + buffer.getOrCreateColumn("l", QwpConstants.TYPE_LONG, true).addLong(1); + buffer.nextRow(); + buffer.reset(); + Assert.assertEquals(1, buffer.getColumnCount()); + assertIllegalState(() -> new QwpSchemaBinding(buffer, known(column("l", ColumnType.LONG)))); + } + } + + @Test + public void testBoundEncodingGuardsDoNotMutateEncoder() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer sentinel = new QwpTableBuffer("s"); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + sentinel.getOrCreateColumn("x", QwpConstants.TYPE_LONG, true).addLong(9); + sentinel.nextRow(); + encoder.encode(sentinel); + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + assertEncoderUnchanged(encoder, () -> encoder.encode(buffer)); + assertEncoderUnchanged(encoder, () -> encoder.encodeWithDeltaDict( + buffer, new GlobalSymbolDictionary(), -1, -1)); + assertEncoderUnchangedIllegalArgument(encoder, () -> encoder.encodeSchema( + buffer, binding.getTableId() + 1, binding.getMetadataVersion())); + binding.longColumn("l", 1); + assertEncoderUnchanged(encoder, () -> encoder.encodeSchema(buffer)); + buffer.nextRow(); + int derivedSize = encoder.encodeSchema(buffer); + byte[] derived = copyFrame(encoder, derivedSize); + int explicitSize = encoder.encodeSchema(buffer, binding.getTableId(), binding.getMetadataVersion()); + Assert.assertEquals(derivedSize, explicitSize); + Assert.assertArrayEquals(derived, copyFrame(encoder, explicitSize)); + + encoder.beginMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertEncoderUnchanged(encoder, () -> encoder.addTable(buffer)); + } + }); + } + + @Test + public void testEmptyBoundEncodingContracts() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + Assert.assertEquals(0, encoder.encodeSchema(buffer)); + Assert.assertEquals(0, encoder.getBuffer().getPosition()); + encoder.beginSchemaMessage(1, new GlobalSymbolDictionary(), -1, -1); + assertEncoderUnchanged(encoder, () -> encoder.addSchemaTable( + buffer, binding.getTableId(), binding.getMetadataVersion())); + } + }); + } + + @Test + public void testUuidConformanceCorpus() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("#")) { + continue; + } + count++; + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 3, fields.length); + CharSequence input = "".equals(fields[1]) ? null : fields[1]; + String expected = fields[2]; + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + if ("".equals(expected)) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, () -> rows.stringColumn("u", input)); + continue; + } + rows.stringColumn("u", input); + buffer.nextRow(); + QwpTableBuffer.ColumnBuffer column = buffer.getColumnCount() == 0 ? null : buffer.getColumn(0); + if ("".equals(expected)) { + // NULL is either a null-bitmap entry or the UUID null sentinel + // (both limbs Long.MIN_VALUE), which the server reads as NULL. + Assert.assertTrue(fields[0], column == null || column.isNull(0) + || (Unsafe.getUnsafe().getLong(column.getDataAddress()) == Long.MIN_VALUE + && Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES) == Long.MIN_VALUE)); + } else { + UUID uuid = UUID.fromString(expected); + Assert.assertNotNull(fields[0], column); + Assert.assertFalse(fields[0], column.isNull(0)); + // The buffer stores the low limb first, as on the wire. + Assert.assertEquals(fields[0], uuid.getLeastSignificantBits(), + Unsafe.getUnsafe().getLong(column.getDataAddress())); + Assert.assertEquals(fields[0], uuid.getMostSignificantBits(), + Unsafe.getUnsafe().getLong(column.getDataAddress() + Long.BYTES)); + } + } + } + } + Assert.assertEquals(20, count); + }); + } + + @Test + public void testStringAndTypedUuidEncodeSameLimbs() throws Exception { + assertMemoryLeak(() -> { + long hi = 0x0123456789abcdefL; + long lo = 0xfedcba9876543210L; + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("u", ColumnType.UUID)); + rows.stringColumn("u", "01234567-89ab-cdef-fedc-ba9876543210"); + buffer.nextRow(); + rows.uuidColumn("u", lo, hi); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 2, QwpConstants.TYPE_UUID); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(lo, reader.i64()); + Assert.assertEquals(hi, reader.i64()); + Assert.assertEquals(lo, reader.i64()); + Assert.assertEquals(hi, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testIpv4ConformanceCorpusUsesExactTargetWire() throws Exception { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_kind\tinput\toutcome\texpected_hex\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 6, fields.length); + for (int targetType : new int[]{ColumnType.IPv4, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("ip", targetType)); + Runnable append = "INT".equals(fields[1]) + ? () -> rows.ipv4Column("ip", Integer.parseInt(fields[2])) + : () -> rows.ipv4Column("ip", fields[2]); + if ("INVALID".equals(fields[3])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + continue; + } + append.run(); + buffer.nextRow(); + byte wireType = targetType == ColumnType.IPv4 + ? QwpConstants.TYPE_IPv4 + : QwpConstants.TYPE_VARCHAR; + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), 1, wireType); + if ("NULL".equals(fields[3])) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + } + } else { + Assert.assertEquals("VALUE", fields[3]); + Assert.assertEquals(0, reader.u8()); + if (wireType == QwpConstants.TYPE_IPv4) { + Assert.assertEquals((int) Long.parseUnsignedLong(fields[4], 16), reader.i32()); + } else { + byte[] expected = fields[5].getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertEquals(fields[5], reader.ascii(expected.length)); + } + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(16, count); + } + + @Test + public void testIpv4DuplicateRollbackNullOmissionAndInference() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("ip", ColumnType.IPv4), column("bad", ColumnType.LONG)); + rows.ipv4Column("ip", 0x01020304).ipv4Column("ip", "not-an-ip"); + buffer.nextRow(); + + rows.ipv4Column("ip", 0x05060708); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("bad", 0x090a0b0c)); + rollbackCurrentRow(buffer); + + rows.ipv4Column("ip", 0); + buffer.nextRow(); + buffer.nextRow(); + rows.ipv4Column("ip", ".0.0.0.0."); + buffer.nextRow(); + rows.ipv4Column("ip", 0x0d0e0f10); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 5, QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x0e, reader.u8()); + Assert.assertEquals(0x01020304, reader.i32()); + Assert.assertEquals(0x0d0e0f10, reader.i32()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.ipv4Column("ignored", (CharSequence) null) + .ipv4Column("ip", "192.168.1.1"); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, + QwpConstants.TYPE_IPv4); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0xc0a80101, reader.i32()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.ipv4Column("ip", 0); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_IPv4, buffer.getColumnDefs()[0].getTypeCode()); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), -1, -1, 1, + QwpConstants.TYPE_IPv4); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + }); + } + + @Test + public void testIpv4RejectsUnsupportedParameterizedAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("long_value", ColumnType.LONG), + column("parameterized", ColumnType.IPv4, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("long_value", 1)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("long_value", "not-an-ip")); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("parameterized", "1.2.3.4")); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.ipv4Column("ts", 1)); + } + } + + @Test + public void testLong256ConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/long256-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_l0\tinput_l1\tinput_l2\tinput_l3\toutcome\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + long l0 = Long.parseLong(fields[1]); + long l1 = Long.parseLong(fields[2]); + long l2 = Long.parseLong(fields[3]); + long l3 = Long.parseLong(fields[4]); + for (int targetType : new int[]{ColumnType.LONG256, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + rows.long256Column("value", l0, l1, l2, l3); + buffer.nextRow(); + byte wireType = targetType == ColumnType.LONG256 + ? QwpConstants.TYPE_LONG256 + : QwpConstants.TYPE_VARCHAR; + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); + if ("NULL".equals(fields[5])) { + Assert.assertEquals("-", fields[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + } + } else { + Assert.assertEquals("VALUE", fields[5]); + Assert.assertEquals(0, reader.u8()); + if (wireType == QwpConstants.TYPE_LONG256) { + Assert.assertEquals(l0, reader.i64()); + Assert.assertEquals(l1, reader.i64()); + Assert.assertEquals(l2, reader.i64()); + Assert.assertEquals(l3, reader.i64()); + } else { + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(fields[6].length(), reader.i32()); + Assert.assertEquals(fields[6], reader.ascii(fields[6].length())); + } + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(8, count); + }); + } + + @Test + public void testLong256DuplicateNullOmissionRollbackAndMissingInference() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", ColumnType.LONG256), column("bad", ColumnType.UUID)); + rows.long256Column("value", 1, 2, 3, 4) + .long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + buffer.nextRow(); + rows.long256Column("value", 5, 6, 7, 8); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("bad", 9, 10, 11, 12)); + rollbackCurrentRow(buffer); + rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x06, reader.u8()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(2, reader.i64()); + Assert.assertEquals(3, reader.i64()); + Assert.assertEquals(4, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.long256Column("value", Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + Assert.assertEquals(1, buffer.getColumnDefs().length); + Assert.assertEquals(QwpConstants.TYPE_LONG256, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_LONG256); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + }); + } + + @Test + public void testLong256DuplicatePrecedesUnsupportedAndRejectsParameterizedDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("bad", ColumnType.UUID), + column("parameterized", ColumnType.LONG256, new byte[]{1})); + rows.uuidColumn("bad", 1, 2) + .long256Column("bad", 3, 4, 5, 6); + buffer.nextRow(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("bad", 3, 4, 5, 6)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("parameterized", 1, 2, 3, 4)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.long256Column("ts", 1, 2, 3, 4)); + } + } + + @Test + public void testNativeGeoHashConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTest.class.getResourceAsStream( + "/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv"); + Assert.assertNotNull(stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + Assert.assertEquals("# case_id\tinput_bits\tprecision_bits\texpected_value_hex\texpected_text", lines.readLine()); + String line; + while ((line = lines.readLine()) != null) { + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 5, fields.length); + long input = Long.parseLong(fields[1]); + int precision = Integer.parseInt(fields[2]); + long expected = Long.parseUnsignedLong(fields[3], 16); + int geoType = ColumnType.getGeoHashTypeWithBits(precision); + for (int targetType : new int[]{geoType, ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, column("value", targetType)); + rows.geoHashColumn("value", input, precision); + buffer.nextRow(); + byte wireType = ColumnType.isGeoHash(targetType) + ? QwpConstants.TYPE_GEOHASH + : QwpConstants.TYPE_VARCHAR; + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, wireType); + if (wireType == QwpConstants.TYPE_GEOHASH) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(precision, reader.varint()); + for (int i = 0; i < (precision + 7) / 8; i++) { + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.u8()); + } + } else { + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(fields[4].length(), reader.i32()); + Assert.assertEquals(fields[4], reader.ascii(fields[4].length())); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ", target=" + + ColumnType.nameOf(targetType) + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertEquals(15, count); + }); + } + + @Test + public void testNativeGeoHashEveryPrecisionUsesFixedWidthText() throws Exception { + assertMemoryLeak(() -> { + for (int precision = 1; precision <= 60; precision++) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + rows(buffer, column("value", ColumnType.STRING)) + .geoHashColumn("value", -1, precision); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(precision, reader.i32()); + for (int i = 0; i < precision; i++) { + Assert.assertEquals('1', reader.u8()); + } + Assert.assertEquals(size, reader.position()); + } + } + }); + } + + @Test + public void testNativeGeoHashOmissionRollbackInferenceAndGuards() throws Exception { + assertMemoryLeak(() -> { + int geo20 = ColumnType.getGeoHashTypeWithBits(20); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", geo20), column("bad", ColumnType.UUID)); + rows.geoHashColumn("value", 0xabcde, 20) + .geoHashColumn("value", 0x12345, 20); + buffer.nextRow(); + buffer.nextRow(); + rows.geoHashColumn("value", 0x11111, 20); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("bad", 1, 20)); + rollbackCurrentRow(buffer); + rows.geoHashColumn("value", 0x12345, 20); + buffer.nextRow(); + + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0x02, reader.u8()); + Assert.assertEquals(20, reader.varint()); + for (long value : new long[]{0xabcde, 0x12345}) { + for (int i = 0; i < 3; i++) { + Assert.assertEquals((int) ((value >>> (i * 8)) & 0xff), reader.u8()); + } + } + Assert.assertEquals(size, reader.position()); + } + + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, schemaResult(QwpSchemaProtocol.RESULT_MISSING)); + rows.geoHashColumn("value", 0xabcde, 20); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_GEOHASH, buffer.getColumnDefs()[0].getTypeCode()); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, -1, -1, 1, QwpConstants.TYPE_GEOHASH); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(20, reader.varint()); + Assert.assertEquals(0xde, reader.u8()); + Assert.assertEquals(0xbc, reader.u8()); + Assert.assertEquals(0x0a, reader.u8()); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("mismatch", ColumnType.getGeoHashTypeWithBits(15)), + column("unsupported", ColumnType.UUID), + column("parameterized", geo20, new byte[]{1})); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("mismatch", 1, 0)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("mismatch", 1, 61)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("mismatch", 1, 20)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("unsupported", 1, 20)); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("parameterized", 1, 20)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = new QwpSchemaBinding(buffer, + known(0, column("ts", ColumnType.TIMESTAMP_MICRO))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("ts", 1, 20)); + } + }); + } + + @Test + public void testNativeGeoHashTextOverloadPreservesSourcePrecision() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("geo", ColumnType.getGeoHashTypeWithBits(20)), + column("text", ColumnType.STRING)); + rows.geoHashColumn("geo", "u33d") + .geoHashColumn("text", "U"); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1, + QwpConstants.TYPE_GEOHASH, QwpConstants.TYPE_VARCHAR); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(20, reader.varint()); + long expected = Numbers.parseGeoHashBase32("u33d"); + for (int i = 0; i < 3; i++) { + Assert.assertEquals((int) ((expected >>> (i * 8)) & 0xff), reader.u8()); + } + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(5, reader.i32()); + Assert.assertEquals("11010", reader.ascii(5)); + Assert.assertEquals(size, reader.position()); + } + + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding rows = rows(buffer, + column("value", ColumnType.getGeoHashTypeWithBits(15))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> rows.geoHashColumn("value", "u33d")); + rollbackCurrentRow(buffer); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", (CharSequence) null)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "")); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "0123456789bcd")); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> rows.geoHashColumn("value", "a")); + } + }); + } + + private static void assertCtorReason(int result, LineSenderSchemaException.Reason reason, String message) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + try { + new QwpSchemaBinding(buffer, schemaResult(result)); + Assert.fail("expected schema exception"); + } catch (LineSenderSchemaException e) { + Assert.assertEquals(reason, e.getReason()); + Assert.assertFalse(e.isRetryable()); + Assert.assertTrue(e.getMessage(), e.getMessage().contains(message)); + } + QwpSchemaBinding binding = rows(buffer, column("l", ColumnType.LONG)); + binding.longColumn("l", 7); + buffer.nextRow(); + assertLongFrame(encoder, encoder.encodeSchema(buffer), 7); + } + } + + private static void assertLongFrame(QwpWebSocketEncoder encoder, int size, long... values) { + QwpTestWireReader reader = tableReader(encoder, size, values.length, QwpConstants.TYPE_LONG); + Assert.assertEquals(0, reader.u8()); + for (long value : values) { + Assert.assertEquals(value, reader.i64()); + } + Assert.assertEquals(size, reader.position()); + } + + private static void appendSmallInteger( + QwpSchemaBinding rows, + String inputType, + long value, + String column + ) { + switch (inputType) { + case "BYTE": + rows.byteColumn(column, (byte) value); + break; + case "SHORT": + rows.shortColumn(column, (short) value); + break; + case "INT": + rows.intColumn(column, (int) value); + break; + default: + throw new AssertionError(inputType); + } + } + + private static int decimalStorageIndex(String target) { + switch (target) { + case "DECIMAL8": + return 0; + case "DECIMAL16": + return 1; + case "DECIMAL32": + return 2; + case "DECIMAL64": + return 3; + case "DECIMAL128": + return 4; + default: + Assert.assertEquals("DECIMAL256", target); + return 5; + } + } + + private static String decimalStorageType(int precision) { + if (precision <= 2) { + return "DECIMAL8"; + } + if (precision <= 4) { + return "DECIMAL16"; + } + if (precision <= 9) { + return "DECIMAL32"; + } + if (precision <= 18) { + return "DECIMAL64"; + } + if (precision <= 38) { + return "DECIMAL128"; + } + return "DECIMAL256"; + } + + private static int decimalWireLongCount(String target) { + if ("DECIMAL128".equals(target)) { + return 2; + } + if ("DECIMAL256".equals(target)) { + return 4; + } + return 1; + } + + private static byte decimalWireType(String target) { + if ("DECIMAL128".equals(target)) { + return QwpConstants.TYPE_DECIMAL128; + } + if ("DECIMAL256".equals(target)) { + return QwpConstants.TYPE_DECIMAL256; + } + return QwpConstants.TYPE_DECIMAL64; + } + + private static void appendIntegerTemporal( + QwpSchemaBinding rows, + String inputType, + long value, + String column + ) { + switch (inputType) { + case "BYTE": + rows.byteColumn(column, (byte) value); + break; + case "SHORT": + rows.shortColumn(column, (short) value); + break; + case "INT": + rows.intColumn(column, (int) value); + break; + case "LONG": + rows.longColumn(column, value); + break; + default: + throw new AssertionError(inputType); + } + } + + private static byte[] hexBytes(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + + private static void assertNumericValue(QwpTestWireReader reader, String target, String expected) { + switch (target) { + case "BYTE": + Assert.assertEquals(Integer.parseInt(expected) & 0xff, reader.u8()); + break; + case "SHORT": + Assert.assertEquals(Integer.parseInt(expected) & 0xffff, reader.u16()); + break; + case "INT": + Assert.assertEquals(Integer.parseInt(expected), reader.i32()); + break; + case "LONG": + Assert.assertEquals(Long.parseLong(expected), reader.i64()); + break; + case "FLOAT": + Assert.assertEquals((int) Long.parseUnsignedLong(expected.substring(2), 16), reader.i32()); + break; + case "DOUBLE": + Assert.assertEquals(Long.parseUnsignedLong(expected.substring(2), 16), reader.i64()); + break; + default: + Assert.fail(target); + } + } + + private static void assertNumericNull(QwpTestWireReader reader, String target) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(target, 1, reader.u8()); + } + + private static void rollbackCurrentRow(QwpTableBuffer buffer) { + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + } + + private static int numericColumnType(String target) { + switch (target) { + case "BYTE": return ColumnType.BYTE; + case "SHORT": return ColumnType.SHORT; + case "INT": return ColumnType.INT; + case "LONG": return ColumnType.LONG; + case "FLOAT": return ColumnType.FLOAT; + case "DOUBLE": return ColumnType.DOUBLE; + default: throw new AssertionError(target); + } + } + + private static long parseHexLong(String value) { + Assert.assertNotEquals("-", value); + return Long.parseUnsignedLong(value, 16); + } + + private static int smallIntegerSourceIndex(String source) { + switch (source) { + case "BYTE": + return 0; + case "SHORT": + return 1; + default: + Assert.assertEquals("INT", source); + return 2; + } + } + + private static int timestampColumnType(String target) { + switch (target) { + case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; + case "TIMESTAMP_NS": return ColumnType.TIMESTAMP_NANO; + default: throw new AssertionError(target); + } + } + + private static int temporalColumnType(String target) { + switch (target) { + case "DATE": return ColumnType.DATE; + case "TIMESTAMP": return ColumnType.TIMESTAMP_MICRO; + case "TIMESTAMP_NS": return ColumnType.TIMESTAMP_NANO; + default: throw new AssertionError(target); + } + } + + private static int textColumnType(String target) { + switch (target) { + case "STRING": return ColumnType.STRING; + case "VARCHAR": return ColumnType.VARCHAR; + case "SYMBOL": return ColumnType.SYMBOL; + default: throw new AssertionError(target); + } + } + + private static byte textWireType(String target) { + return "SYMBOL".equals(target) ? QwpConstants.TYPE_SYMBOL : QwpConstants.TYPE_VARCHAR; + } + + private static byte timestampWireType(String target) { + switch (target) { + case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; + case "TIMESTAMP_NS": return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: throw new AssertionError(target); + } + } + + private static byte temporalWireType(String target) { + switch (target) { + case "DATE": return QwpConstants.TYPE_DATE; + case "TIMESTAMP": return QwpConstants.TYPE_TIMESTAMP; + case "TIMESTAMP_NS": return QwpConstants.TYPE_TIMESTAMP_NANOS; + default: throw new AssertionError(target); + } + } + + private static byte numericWireType(String target) { + switch (target) { + case "BYTE": return QwpConstants.TYPE_BYTE; + case "SHORT": return QwpConstants.TYPE_SHORT; + case "INT": return QwpConstants.TYPE_INT; + case "LONG": return QwpConstants.TYPE_LONG; + case "FLOAT": return QwpConstants.TYPE_FLOAT; + case "DOUBLE": return QwpConstants.TYPE_DOUBLE; + default: throw new AssertionError(target); + } + } + + private static byte[] copyFrame(QwpWebSocketEncoder encoder, int size) { + byte[] copy = new byte[size]; + long address = encoder.getBuffer().getBufferPtr(); + for (int i = 0; i < size; i++) { + copy[i] = Unsafe.getUnsafe().getByte(address + i); + } + return copy; + } + + private static void assertIllegalArgument(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertIllegalState(Runnable action) { + try { + action.run(); + Assert.fail("expected IllegalStateException"); + } catch (IllegalStateException expected) { + // expected + } + } + + private static void assertEncoderUnchanged(QwpWebSocketEncoder encoder, Runnable action) { + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + assertIllegalState(action); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + + private static void assertEncoderUnchangedIllegalArgument(QwpWebSocketEncoder encoder, Runnable action) { + int position = encoder.getBuffer().getPosition(); + byte[] before = copyFrame(encoder, position); + assertIllegalArgument(action); + Assert.assertEquals(position, encoder.getBuffer().getPosition()); + Assert.assertArrayEquals(before, copyFrame(encoder, position)); + } + + private static QwpSchemaBinding rows(QwpTableBuffer buffer, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(columns)); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte... wireTypes) { + return tableReader(encoder, size, 1, 1, rows, wireTypes); + } + + private static QwpTestWireReader tableReader( + QwpWebSocketEncoder encoder, + int size, + int tableId, + long metadataVersion, + int rows, + byte... wireTypes + ) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + Assert.assertEquals("t", reader.string()); + if (tableId < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(tableId, reader.i32()); + Assert.assertEquals(metadataVersion, reader.i64()); + } + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(wireTypes.length, reader.varint()); + for (byte wireType : wireTypes) { + reader.string(); + Assert.assertEquals(wireType, reader.u8()); + } + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java new file mode 100644 index 000000000..5ff9bba26 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTextTest.java @@ -0,0 +1,354 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.Sender; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.skipTableHeader; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/text-to-text.tsv"; + + @Test + public void testConformanceCorpusUsesExactTargetWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\\t", -1); + Assert.assertEquals(line, 7, fields.length); + try { + String value = "".equals(fields[3]) ? null : utf16(fields[3]); + byte wireType = wireType(fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[2]))); + append(binding, fields[1], value); + buffer.nextRow(); + QwpTestWireReader reader = tableReader(encoder, encoder.encodeSchema(buffer), wireType); + if (value == null) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + if (wireType == QwpConstants.TYPE_VARCHAR) { + Assert.assertEquals(0, reader.i32()); + } else { + Assert.assertEquals(0, reader.varint()); + } + } else if (wireType == QwpConstants.TYPE_VARCHAR) { + byte[] expected = hex(fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertArrayEquals(expected, reader.stringBytes()); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(utf16(fields[6]), new String(expected, StandardCharsets.UTF_8)); + } + Assert.assertEquals(encoder.getBuffer().getPosition(), reader.position()); + } + count++; + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + } + } + Assert.assertEquals(90, count); + }); + } + + @Test + public void testFirstValueNullOmissionAndMutableInput() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("text", ColumnType.STRING), column("sym", ColumnType.SYMBOL)); + StringBuilder mutable = new StringBuilder("before"); + binding.symbol("text", mutable).stringColumn("text", "ignored"); + binding.stringColumn("sym", "first").symbol("sym", null); + mutable.replace(0, mutable.length(), "after"); + buffer.nextRow(); + binding.stringColumn("text", null).symbol("sym", null); + buffer.nextRow(); + buffer.nextRow(); + Assert.assertEquals("before".length(), buffer.getColumn(0).getStringDataSize()); + Assert.assertArrayEquals("before".getBytes(StandardCharsets.UTF_8), + bytes(buffer.getColumn(0).getStringDataAddress(), "before".length())); + Assert.assertEquals("first", buffer.getColumn(1).getSymbolValue(0).toString()); + int size = encoder.encodeSchema(buffer); + Assert.assertTrue(size > QwpConstants.HEADER_SIZE); + Assert.assertEquals(3, buffer.getRowCount()); + } + }); + } + + @Test + public void testFailureRollbackAndLifecycle() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaResponse schema = known(-1, + column("a", ColumnType.STRING), column("only_b", ColumnType.SYMBOL), + column("bad", ColumnType.UUID), column("c", ColumnType.VARCHAR)); + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, schema); + binding.stringColumn("a", "A"); + buffer.nextRow(); + binding.symbol("only_b", "B"); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("bad", "invalid")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.symbol("c", "C"); + buffer.nextRow(); + Assert.assertEquals(2, buffer.getColumnCount()); + Assert.assertEquals("a", buffer.getColumnDefs()[0].getName()); + Assert.assertEquals("c", buffer.getColumnDefs()[1].getName()); + Assert.assertTrue(encoder.encodeSchema(buffer) > 0); + + buffer.reset(); + binding.symbol("a", "reset"); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", new String(reset.stringBytes(), StandardCharsets.UTF_8)); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(5, reset.i32()); + Assert.assertArrayEquals("reset".getBytes(StandardCharsets.UTF_8), reset.bytes(5)); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + buffer.clear(); + assertIllegalState(() -> binding.symbol("a", "stale")); + assertIllegalState(() -> binding.stringColumn("a", "stale")); + } + }); + } + + @Test + public void testUnsupportedParameterizedUnknownAndDesignatedTargets() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("long", ColumnType.LONG), column("future", ColumnType.SYMBOL, new byte[]{1}), + column("flagged", ColumnType.STRING | 0x10000)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("long", null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("future", "x")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("flagged", "x")); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.symbol("missing", "x"); + Assert.assertEquals(QwpConstants.TYPE_SYMBOL, + buffer.getColumnDefs()[buffer.getColumnCount() - 1].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, known(0, column("d", ColumnType.SYMBOL))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.symbol("d", null)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.stringColumn("d", null)); + } + } + + @Test + public void testUnsupportedDuplicateIsIgnoredAcrossTextSetters() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("uuid", ColumnType.UUID), column("long", ColumnType.LONG)); + binding.stringColumn("uuid", "01234567-89ab-cdef-0123-456789abcdef") + .symbol("uuid", null); + binding.longColumn("long", 7).symbol("long", null); + buffer.nextRow(); + Assert.assertTrue(encoder.encodeSchema(buffer) > 0); + } + }); + } + + @Test + public void testOwnerBackedGlobalSymbolIdsAndEncoding() throws Exception { + assertMemoryLeak(() -> { + TestWebSocketServer.WebSocketServerHandler handler = new TestWebSocketServer.WebSocketServerHandler() { }; + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + try (Sender senderApi = Sender.fromConfig("ws::addr=localhost:" + server.getPort() + + ";close_flush_timeout_millis=0;")) { + QwpWebSocketSender sender = (QwpWebSocketSender) senderApi; + Assert.assertEquals(0, sender.getOrAddGlobalSymbol("prefix")); + GlobalSymbolDictionary mirror = new GlobalSymbolDictionary(); + Assert.assertEquals(0, mirror.getOrAddSymbol("prefix")); + try (QwpTableBuffer buffer = new QwpTableBuffer("t", sender); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, column("value", ColumnType.SYMBOL)); + StringBuilder mutable = new StringBuilder("mutable"); + binding.symbol("value", mutable); + mutable.replace(0, mutable.length(), "changed"); + buffer.nextRow(); + binding.stringColumn("value", "?"); + buffer.nextRow(); + binding.symbol("value", "\ud800"); + buffer.nextRow(); + binding.symbol("value", "mutable"); + buffer.nextRow(); + + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("mutable")); + Assert.assertEquals(2, sender.getOrAddGlobalSymbol("?")); + Assert.assertEquals(3, sender.getOrAddGlobalSymbol("\ud800")); + Assert.assertEquals(1, sender.getOrAddGlobalSymbol("mutable")); + mirror.getOrAddSymbol("mutable"); + mirror.getOrAddSymbol("?"); + mirror.getOrAddSymbol("\ud800"); + Assert.assertEquals(0, buffer.getColumn(0).getSymbolDictionarySize()); + + encoder.beginSchemaMessage(1, mirror, -1, 3); + encoder.addSchemaTable(buffer, binding.getTableId(), binding.getMetadataVersion()); + int size = encoder.finishMessage(); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals(0, reader.varint()); + Assert.assertEquals(4, reader.varint()); + Assert.assertArrayEquals("prefix".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals("mutable".getBytes(StandardCharsets.UTF_8), reader.stringBytes()); + Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); + Assert.assertArrayEquals(new byte[]{'?'}, reader.stringBytes()); + skipTableHeader(reader, QwpConstants.TYPE_SYMBOL, 4); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(size, reader.position()); + } + } + } + }); + } + + private static void append(QwpSchemaBinding binding, String inputType, CharSequence value) { + if ("STRING".equals(inputType)) { + binding.stringColumn("value", value); + } else if ("SYMBOL".equals(inputType)) { + binding.symbol("value", value); + } else { + throw new AssertionError("unknown input type: " + inputType); + } + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, byte wireType) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + skipTableHeader(reader, wireType, 1); + return reader; + } + + private static int targetType(String value) { + switch (value) { + case "STRING": + return ColumnType.STRING; + case "VARCHAR": + return ColumnType.VARCHAR; + case "SYMBOL": + return ColumnType.SYMBOL; + default: + throw new AssertionError("unknown target type: " + value); + } + } + + private static byte wireType(String value) { + switch (value) { + case "VARCHAR": + return QwpConstants.TYPE_VARCHAR; + case "SYMBOL": + return QwpConstants.TYPE_SYMBOL; + default: + throw new AssertionError("unknown wire type: " + value); + } + } + + private static byte[] hex(String value) { + Assert.assertEquals(0, value.length() & 1); + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + } + return bytes; + } + + private static byte[] bytes(long address, int length) { + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = Unsafe.getUnsafe().getByte(address + i); + } + return value; + } + + private static String utf16(String value) { + Assert.assertEquals(0, value.length() & 3); + char[] chars = new char[value.length() / 4]; + for (int i = 0; i < chars.length; i++) { + chars[i] = (char) Integer.parseInt(value.substring(i * 4, i * 4 + 4), 16); + } + return new String(chars); + } + + private static void assertIllegalState(Runnable action) { + Assert.assertThrows(IllegalStateException.class, action::run); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java new file mode 100644 index 000000000..88e2cdf25 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingTimestampTextTest.java @@ -0,0 +1,210 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.temporal.ChronoUnit; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.tableHeader; + +public class QwpSchemaBindingTimestampTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv"; + + @Test + public void testCorpusUsesExactVarcharBytes() throws Exception { + InputStream stream = getClass().getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + boolean headerSeen = false; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + if (line.charAt(0) == '#') { + Assert.assertFalse("duplicate corpus header", headerSeen); + Assert.assertEquals( + "# case_id\tinput_kind\tunit\tvalue\tnanos\toutcome\texpected_text", line); + headerSeen = true; + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertTrue("LONG".equals(fields[1]) || "INSTANT".equals(fields[1])); + Assert.assertTrue("VALUE".equals(fields[5]) || "INVALID".equals(fields[5])); + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", target)); + Runnable append = () -> append(binding, fields); + if ("INVALID".equals(fields[5])) { + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, append); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + continue; + } + append.run(); + buffer.nextRow(); + QwpTestWireReader reader = tableReader( + encoder, encoder.encodeSchema(buffer), 1, QwpConstants.TYPE_VARCHAR); + String expectedText = "".equals(fields[6]) ? "" : fields[6]; + byte[] expected = expectedText.getBytes(StandardCharsets.UTF_8); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + Assert.assertEquals(reader.limit(), reader.position()); + } catch (AssertionError e) { + throw new AssertionError( + "case_id=" + fields[0] + ", target=" + target + ": " + e.getMessage(), e); + } + } + count++; + } + } + Assert.assertTrue("missing corpus header", headerSeen); + Assert.assertEquals(63, count); + } + + private static void append(QwpSchemaBinding binding, String[] fields) { + if ("INSTANT".equals(fields[1])) { + binding.timestampColumn("value", Instant.ofEpochSecond( + Long.parseLong(fields[3]), Integer.parseInt(fields[4]))); + } else if ("LONG".equals(fields[1])) { + binding.timestampColumn("value", Long.parseLong(fields[3]), ChronoUnit.valueOf(fields[2])); + } else { + throw new AssertionError("unknown input kind: " + fields[1]); + } + } + + @Test + public void testUnitsInstantNullDuplicateOmissionRollbackAndReset() { + for (int target : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpTableBuffer buffer = new QwpTableBuffer("t"); + QwpWebSocketEncoder encoder = new QwpWebSocketEncoder()) { + QwpSchemaBinding binding = binding(buffer, -1, + column("a", target), column("bad", ColumnType.UUID), column("c", target)); + binding.timestampColumn("a", Long.MIN_VALUE, ChronoUnit.MICROS) + .binaryColumn("a", new byte[]{1}); + buffer.nextRow(); + binding.timestampColumn("a", -1, ChronoUnit.NANOS); + buffer.nextRow(); + buffer.nextRow(); + binding.timestampColumn("a", Instant.ofEpochSecond(-1, 999_999_999)); + buffer.nextRow(); + binding.timestampColumn("a", 1, ChronoUnit.DAYS); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("bad", 1, ChronoUnit.MICROS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.timestampColumn("c", 1000, ChronoUnit.NANOS); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableHeader(encoder, size, 5, 2); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarchar(reader, 0x14, "", "1970-01-01T00:00:00.000Z", "1969-12-31T23:59:59.999Z"); + assertVarchar(reader, 0x0f, "1970-01-01T00:00:00.000Z"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.timestampColumn("a", 0, ChronoUnit.SECONDS); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = tableHeader(encoder, resetSize, 1, 2); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + assertVarchar(reset, -1, "1970-01-01T00:00:00.000Z"); + assertVarchar(reset, 0x01); + Assert.assertEquals(resetSize, reset.position()); + } + } + } + + @Test + public void testInferenceDesignatedAndRangeErrors() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, missing()); + binding.timestampColumn("value", Long.MIN_VALUE, ChronoUnit.NANOS); + buffer.nextRow(); + Assert.assertEquals(QwpConstants.TYPE_TIMESTAMP_NANOS, buffer.getColumnDefs()[0].getTypeCode()); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, 0, column("ts", ColumnType.TIMESTAMP_MICRO)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("ts", 1, ChronoUnit.MICROS)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, -1, column("value", ColumnType.VARCHAR)); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", (Instant) null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.INVALID_VALUE, + () -> binding.timestampColumn("value", 1, null)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.timestampColumn("value", 1, ChronoUnit.MONTHS)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.timestampColumn("value", 1, ChronoUnit.MICROS); + binding.timestampColumn("value", Long.MAX_VALUE, ChronoUnit.DAYS); + buffer.nextRow(); + } + } + + private static void assertVarchar(QwpTestWireReader reader, int bitmap, String... values) { + if (bitmap < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + } + int total = 0; + StringBuilder joined = new StringBuilder(); + Assert.assertEquals(0, reader.i32()); + for (String value : values) { + total += value.getBytes(StandardCharsets.UTF_8).length; + Assert.assertEquals(total, reader.i32()); + joined.append(value); + } + Assert.assertArrayEquals(joined.toString().getBytes(StandardCharsets.UTF_8), reader.bytes(total)); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, byte type) { + QwpTestWireReader reader = tableHeader(encoder, size, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(type, reader.u8()); + return reader; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java new file mode 100644 index 000000000..680b590c5 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaBindingUuidTextTest.java @@ -0,0 +1,249 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cairo.ColumnType; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.assertReason; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.binding; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.known; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.missing; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.parameterizedColumn; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class QwpSchemaBindingUuidTextTest { + private static final String CORPUS = "/io/questdb/client/cutlass/qwp/uuid-to-text.tsv"; + + @Test + public void testCorpusUsesCanonicalVarcharWire() throws Exception { + assertMemoryLeak(() -> { + InputStream stream = QwpSchemaBindingUuidTextTest.class.getResourceAsStream(CORPUS); + Assert.assertNotNull(CORPUS, stream); + int count = 0; + try (BufferedReader lines = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + String line; + while ((line = lines.readLine()) != null) { + if (line.isEmpty() || line.charAt(0) == '#') { + continue; + } + String[] fields = line.split("\t", -1); + Assert.assertEquals(line, 7, fields.length); + Assert.assertEquals("VARCHAR", fields[4]); + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType(fields[3]))); + binding.uuidColumn("value", Long.parseLong(fields[1]), Long.parseLong(fields[2])); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 1); + if ("".equals(fields[5])) { + Assert.assertEquals("", fields[6]); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(0, reader.i32()); + } else { + byte[] expected = hex(fields[5]); + Assert.assertEquals(fields[6], new String(expected, StandardCharsets.UTF_8)); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(expected.length, reader.i32()); + Assert.assertArrayEquals(expected, reader.bytes(expected.length)); + } + Assert.assertEquals(size, reader.position()); + } catch (AssertionError e) { + throw new AssertionError("case_id=" + fields[0] + ": " + e.getMessage(), e); + } + count++; + } + } + Assert.assertEquals(16, count); + }); + } + + @Test + public void testDuplicateNullOmissionRollbackAndReset() throws Exception { + assertMemoryLeak(() -> { + for (int targetType : new int[]{ColumnType.STRING, ColumnType.VARCHAR}) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, column("value", targetType)); + binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE) + .binaryColumn("value", new byte[]{1}) + .longColumn("value", 42); + buffer.nextRow(); + binding.uuidColumn("value", 1, 0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = tableReader(encoder, size, 3); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(5, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(36, reader.i32()); + Assert.assertEquals("00000000-0000-0000-0000-000000000001", reader.ascii(36)); + Assert.assertEquals(size, reader.position()); + } + } + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = binding(buffer, + column("a", ColumnType.STRING), column("bad", ColumnType.LONG), column("c", ColumnType.VARCHAR)); + binding.uuidColumn("a", 1, 0); + buffer.nextRow(); + binding.uuidColumn("a", 2, 0); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> binding.uuidColumn("bad", 3, 0)); + buffer.cancelCurrentRow(); + buffer.rollbackUncommittedColumns(); + binding.uuidColumn("c", 4, 0); + buffer.nextRow(); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(3, reader.varint()); + Assert.assertEquals(2, reader.varint()); + Assert.assertEquals("a", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + Assert.assertEquals("c", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + assertVarcharValues(reader, 6, "00000000-0000-0000-0000-000000000001"); + assertVarcharValues(reader, 5, "00000000-0000-0000-0000-000000000004"); + Assert.assertEquals(size, reader.position()); + + buffer.reset(); + binding.uuidColumn("a", 5, 0); + buffer.nextRow(); + int resetSize = encoder.encodeSchema(buffer); + QwpTestWireReader reset = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), resetSize); + reset.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reset.string()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.i32()); + Assert.assertEquals(1, reset.i64()); + Assert.assertEquals(1, reset.varint()); + Assert.assertEquals(2, reset.varint()); + Assert.assertEquals("a", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals("c", reset.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reset.u8()); + Assert.assertEquals(0, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(36, reset.i32()); + Assert.assertEquals("00000000-0000-0000-0000-000000000005", reset.ascii(36)); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(1, reset.u8()); + Assert.assertEquals(0, reset.i32()); + Assert.assertEquals(resetSize, reset.position()); + } + }); + } + + @Test + public void testNativeInferenceAndNativeBothMinRemainUuidBytes() { + assertNativeBothMin(known(column("value", ColumnType.UUID)), true); + assertNativeBothMin(missing(), false); + } + + private static void assertNativeBothMin(QwpSchemaResponse schema, boolean known) { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding binding = new QwpSchemaBinding(buffer, schema); + binding.uuidColumn("value", Long.MIN_VALUE, Long.MIN_VALUE); + buffer.nextRow(); + int size = encoder.encodeSchema(buffer); + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(known ? 1 : 0, reader.u8()); + if (known) { + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + } + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(QwpConstants.TYPE_UUID, reader.u8()); + Assert.assertEquals(0, reader.u8()); + Assert.assertEquals(Long.MIN_VALUE, reader.i64()); + Assert.assertEquals(Long.MIN_VALUE, reader.i64()); + Assert.assertEquals(size, reader.position()); + } + } + + @Test + public void testSymbolParameterizedAndDesignatedTargetsRemainUnsupported() { + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding ordinary = binding(buffer, + column("sym", ColumnType.SYMBOL), parameterizedColumn("text", ColumnType.STRING), + column("ts", ColumnType.TIMESTAMP)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> ordinary.uuidColumn("sym", Long.MIN_VALUE, Long.MIN_VALUE)); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, + () -> ordinary.uuidColumn("text", 1, 0)); + } + try (QwpTableBuffer buffer = new QwpTableBuffer("t")) { + QwpSchemaBinding designated = new QwpSchemaBinding(buffer, known(0, column("ts", ColumnType.TIMESTAMP))); + assertReason(LineSenderSchemaException.Reason.UNSUPPORTED_FEATURE, () -> designated.uuidColumn("ts", 1, 0)); + } + } + + private static void assertVarcharValues(QwpTestWireReader reader, int bitmap, String value) { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(bitmap, reader.u8()); + Assert.assertEquals(0, reader.i32()); + Assert.assertEquals(36, reader.i32()); + Assert.assertEquals(value, reader.ascii(36)); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows) { + return tableReader(encoder, size, rows, "value"); + } + + private static QwpTestWireReader tableReader(QwpWebSocketEncoder encoder, int size, int rows, String column) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + reader.skip(QwpConstants.HEADER_SIZE); + Assert.assertEquals("t", reader.string()); + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(1, reader.i32()); + Assert.assertEquals(1, reader.i64()); + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(1, reader.varint()); + Assert.assertEquals(column, reader.string()); + Assert.assertEquals(QwpConstants.TYPE_VARCHAR, reader.u8()); + return reader; + } + + private static int targetType(String value) { + if ("STRING".equals(value)) return ColumnType.STRING; + if ("VARCHAR".equals(value)) return ColumnType.VARCHAR; + throw new AssertionError(value); + } + + private static byte[] hex(String value) { + byte[] bytes = new byte[value.length() / 2]; + for (int i = 0; i < bytes.length; i++) bytes[i] = (byte) Integer.parseInt(value.substring(i * 2, i * 2 + 2), 16); + return bytes; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java new file mode 100644 index 000000000..ef1aad876 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaProtocolTest.java @@ -0,0 +1,329 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.column; +import static io.questdb.client.test.cutlass.qwp.protocol.QwpSchemaTestFixtures.frame; + +public class QwpSchemaProtocolTest { + @Test + public void testAcceptsMaximumResponseColumnNameBoundaries() { + String name = repeat('\u0800', QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH); + QwpSchemaResponse response = decode(knownResponse(1, 1, 1, 0, column(name, 5, new byte[0]))); + Assert.assertEquals(name, response.getColumnName(0)); + Assert.assertEquals(QwpSchemaProtocol.MAX_NAME_UTF8_LENGTH, + name.getBytes(StandardCharsets.UTF_8).length); + } + + @Test + public void testDecodeKnownSchemaAndSkipsUnknownParams() { + byte[] frame = knownResponse(7, 42, 19, 1, + column("id", 32, new byte[]{1, 2, 3}), + column("métric🚀", 0x71234567, new byte[0])); + QwpSchemaResponse response = decode(frame); + Assert.assertEquals(7, response.getRequestId()); + Assert.assertEquals(QwpSchemaProtocol.RESULT_KNOWN, response.getResult()); + Assert.assertTrue(response.hasSchema()); + Assert.assertEquals(42, response.getTableId()); + Assert.assertEquals(19, response.getMetadataVersion()); + Assert.assertEquals(1, response.getDesignatedIndex()); + Assert.assertEquals(2, response.getColumnCount()); + Assert.assertTrue(response.hasColumnExtensionParameters(0)); + Assert.assertFalse(response.hasColumnExtensionParameters(1)); + Assert.assertEquals("métric🚀", response.getColumnName(1)); + Assert.assertEquals(0x71234567, response.getColumnType(1)); + } + + @Test + public void testRejectsInvalidAndDuplicateColumnNames() { + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[0]), column("x", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[0]), column("X", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("Ä", 5, new byte[0]), column("ä", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("a/b", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("a\nb", 5, new byte[0]))); + } + + @Test + public void testAcceptsValidColumnNameCompatibilityBoundaries() { + String max = repeat('\u0800', QwpSchemaProtocol.MAX_NAME_UTF16_LENGTH); + QwpSchemaResponse response = decode(knownResponse(1, 1, 0, -1, + column("metric name", 5, new byte[0]), + column("метрика", 5, new byte[0]), + column("!^[$", 5, new byte[0]), + column(max, 5, new byte[0]))); + Assert.assertEquals(4, response.getColumnCount()); + Assert.assertEquals(max, response.getColumnName(3)); + } + + @Test + public void testDecodeEverySchemaLessResult() { + for (int result = QwpSchemaProtocol.RESULT_MISSING; result <= QwpSchemaProtocol.RESULT_TOO_LARGE; result++) { + QwpSchemaResponse response = decode(resultResponse(11, result)); + Assert.assertEquals(result, response.getResult()); + Assert.assertFalse(response.hasSchema()); + Assert.assertEquals(0, response.getColumnCount()); + Assert.assertEquals(-1, response.getTableId()); + } + } + + @Test + public void testEncodeDescribeExactAsciiWire() { + byte[] encoded = QwpSchemaProtocol.encodeDescribe(0x0102030405060708L, "trades"); + ByteBuffer b = ByteBuffer.wrap(encoded).order(ByteOrder.LITTLE_ENDIAN); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, b.getInt()); + Assert.assertEquals(1, b.get() & 0xff); + Assert.assertEquals(QwpSchemaProtocol.FLAG_CONTROL, b.get()); + Assert.assertEquals(0, b.getShort()); + Assert.assertEquals(encoded.length - QwpConstants.HEADER_SIZE, b.getInt()); + Assert.assertEquals(QwpSchemaProtocol.KIND_DESCRIBE, b.get()); + Assert.assertEquals(0x0102030405060708L, b.getLong()); + Assert.assertEquals(6, b.getShort()); + byte[] name = new byte[6]; + b.get(name); + Assert.assertArrayEquals("trades".getBytes(StandardCharsets.UTF_8), name); + Assert.assertFalse(b.hasRemaining()); + } + + @Test + public void testEncodeDescribeUnicodeBoundariesAndRejectsInvalidNames() { + String maxUtf16AndUtf8 = repeat('\u0800', 127); + byte[] encoded = QwpSchemaProtocol.encodeDescribe(1, maxUtf16AndUtf8); + Assert.assertEquals(381, Short.toUnsignedInt(ByteBuffer.wrap(encoded, 21, 2).order(ByteOrder.LITTLE_ENDIAN).getShort())); + + assertEncodeRejected(0, "table"); + assertEncodeRejected(1, ""); + assertEncodeRejected(1, repeat('a', 128)); + assertEncodeRejected(1, repeat('\u0800', 126) + "🚀"); // 128 UTF-16 units + assertEncodeRejected(1, "bad\uD800name"); + assertEncodeRejected(1, "bad\uDC00name"); + assertEncodeRejected(1, "bad/name"); + } + + @Test + public void testGetColumnIndexIsCaseInsensitive() { + QwpSchemaResponse known = decode(knownResponse(1, 1, 1, 1, + column("Id", 5, new byte[0]), + column("métric🚀", 6, new byte[0]))); + Assert.assertEquals(0, known.getColumnIndex("id")); + Assert.assertEquals(0, known.getColumnIndex("ID")); + Assert.assertEquals(1, known.getColumnIndex("MÉTRIC🚀")); + Assert.assertEquals(-1, known.getColumnIndex("absent")); + + QwpSchemaResponse missing = decode(resultResponse(1, QwpSchemaProtocol.RESULT_MISSING)); + Assert.assertEquals(-1, missing.getColumnIndex("id")); + } + + @Test + public void testRejectsMalformedEnvelopesAndTrailingData() { + byte[] valid = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); + assertMalformed(mutate(valid, 0, (byte) 'X')); + assertMalformed(mutate(valid, 4, (byte) 2)); + assertMalformed(mutate(valid, 5, (byte) (QwpSchemaProtocol.FLAG_CONTROL | 1))); + assertMalformed(mutate(valid, 6, (byte) 1)); + assertMalformed(mutate(valid, 12, QwpSchemaProtocol.KIND_DESCRIBE)); + assertMalformed(Arrays.copyOf(valid, valid.length - 1)); + + byte[] trailing = Arrays.copyOf(valid, valid.length + 1); + ByteBuffer.wrap(trailing).order(ByteOrder.LITTLE_ENDIAN).putInt(8, trailing.length - 12); + assertMalformed(trailing); + byte[] oversized = new byte[QwpSchemaProtocol.MAX_FRAME_SIZE + 1]; + assertMalformed(oversized); + } + + @Test + public void testPeekResponseRequestIdValidatesOnlyFixedEnvelopeAndIdentity() { + byte[] valid = resultResponse(7, QwpSchemaProtocol.RESULT_MISSING); + byte[] noResult = responseIdentity(7); + Assert.assertEquals(7, peek(noResult)); + Assert.assertEquals(7, peek(valid)); + assertMalformed(Arrays.copyOf(noResult, 20), true); + assertMalformed(mutate(valid, 0, (byte) 'X'), true); + assertMalformed(mutate(valid, 4, (byte) 2), true); + assertMalformed(mutate(valid, 5, (byte) (QwpSchemaProtocol.FLAG_CONTROL | 1)), true); + assertMalformed(mutate(valid, 6, (byte) 1), true); + assertMalformed(mutate(valid, 12, QwpSchemaProtocol.KIND_DESCRIBE), true); + assertMalformed(resultResponse(0, QwpSchemaProtocol.RESULT_MISSING), true); + + byte[] badLength = valid.clone(); + ByteBuffer.wrap(badLength).order(ByteOrder.LITTLE_ENDIAN).putInt(8, 9); + assertMalformed(badLength, true); + + // Identification deliberately stops before the result/body, while full decoding stays strict. + assertMalformed(noResult); + } + + @Test + public void testRejectsEveryTruncatedKnownResponsePrefix() { + byte[] valid = knownResponse(9, 7, 6, 1, + column("first", 5, new byte[]{1, 2, 3}), + column("секунд", 0x71234567, new byte[0])); + Assert.assertEquals(2, decode(valid).getColumnCount()); + for (int length = 0; length < valid.length; length++) { + byte[] prefix = Arrays.copyOf(valid, length); + if (length >= QwpConstants.HEADER_SIZE) { + ByteBuffer.wrap(prefix).order(ByteOrder.LITTLE_ENDIAN) + .putInt(8, length - QwpConstants.HEADER_SIZE); + } + assertMalformed(prefix); + } + } + + @Test + public void testRejectsInvalidResultsIdentitiesCountsAndIndexes() { + assertMalformed(resultResponse(0, QwpSchemaProtocol.RESULT_MISSING)); + assertMalformed(resultResponse(1, 5)); + assertMalformed(knownResponse(1, -1, 0, -1)); + assertMalformed(knownResponse(1, 1, -1, -1)); + assertMalformed(knownResponse(1, 1, 0, 0)); + assertMalformed(knownResponse(1, 1, 0, -2)); + + byte[] count = knownResponse(1, 1, 0, -1); + ByteBuffer.wrap(count).order(ByteOrder.LITTLE_ENDIAN).putShort(36, (short) 2049); + assertMalformed(count); + } + + @Test + public void testRejectsInvalidUtf8AndFieldBoundsBeforeAllocation() { + byte[] invalidUtf8 = knownResponse(1, 1, 0, -1, columnBytes(new byte[]{(byte) 0xc3, 0x28}, 5, new byte[0])); + assertMalformed(invalidUtf8); + assertMalformed(knownResponse(1, 1, 0, -1, column("", 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column(repeat('a', 128), 5, new byte[0]))); + assertMalformed(knownResponse(1, 1, 0, -1, column("x", 5, new byte[1025]))); + + byte[] truncatedName = knownResponse(1, 1, 0, -1, column("abc", 5, new byte[0])); + ByteBuffer.wrap(truncatedName).order(ByteOrder.LITTLE_ENDIAN).putShort(38, (short) 381); + assertMalformed(truncatedName); + byte[] truncatedParams = knownResponse(1, 1, 0, -1, column("x", 5, new byte[0])); + ByteBuffer.wrap(truncatedParams).order(ByteOrder.LITTLE_ENDIAN).putShort(45, (short) 1024); + assertMalformed(truncatedParams); + } + + private static void assertEncodeRejected(long requestId, String name) { + try { + QwpSchemaProtocol.encodeDescribe(requestId, name); + Assert.fail("expected invalid describe request"); + } catch (IllegalArgumentException expected) { + // expected + } + } + + private static void assertMalformed(byte[] frame) { + try { + decode(frame); + Assert.fail("expected malformed frame"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Malformed QWP schema response:")); + } + } + + private static void assertMalformed(byte[] frame, boolean peek) { + try { + if (peek) { + peek(frame); + } else { + decode(frame); + } + Assert.fail("expected malformed frame"); + } catch (IllegalArgumentException expected) { + Assert.assertTrue(expected.getMessage().startsWith("Malformed QWP schema response:")); + } + } + + private static byte[] columnBytes(byte[] name, int type, byte[] params) { + ByteBuffer b = ByteBuffer.allocate(2 + name.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN); + b.putShort((short) name.length).put(name).putInt(type).putShort((short) params.length).put(params); + return b.array(); + } + + private static QwpSchemaResponse decode(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static long peek(byte[] frame) { + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.peekResponseRequestId(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + private static byte[] knownResponse(long requestId, int tableId, long version, int designated, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) length += column.length; + ByteBuffer b = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(tableId).putLong(version).putShort((short) designated).putShort((short) columns.length); + for (byte[] column : columns) b.put(column); + return frame(b.array()); + } + + private static byte[] mutate(byte[] source, int offset, byte value) { + byte[] copy = source.clone(); + copy[offset] = value; + return copy; + } + + private static String repeat(char c, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + private static byte[] resultResponse(long requestId, int result) { + ByteBuffer b = ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId).put((byte) result); + return frame(b.array()); + } + + private static byte[] responseIdentity(long requestId) { + ByteBuffer b = ByteBuffer.allocate(9).order(ByteOrder.LITTLE_ENDIAN); + b.put(QwpSchemaProtocol.KIND_SCHEMA).putLong(requestId); + return frame(b.array()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java new file mode 100644 index 000000000..b4c3e6de8 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpSchemaTestFixtures.java @@ -0,0 +1,190 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.LineSenderSchemaException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaBinding; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaProtocol; +import io.questdb.client.cutlass.qwp.protocol.QwpSchemaResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +/** + * Shared mechanics for the {@code QwpSchemaBinding*Test} family: schema + * response construction and encoded-table header checks. The fixtures write + * the schema envelope by hand, independently of any production encoder, and + * leave every conversion expectation to the individual tests. + * + *

      All fixtures use table {@code "t"}, table id 1 and metadata version 1 + * unless a parameter says otherwise.

      + */ +final class QwpSchemaTestFixtures { + private static final byte[] NO_PARAMS = new byte[0]; + + private QwpSchemaTestFixtures() { + } + + /** Asserts a non-retryable schema rejection whose message contains every part. */ + static void assertReason(LineSenderSchemaException.Reason reason, Runnable action, String... messageParts) { + LineSenderSchemaException error = Assert.assertThrows(LineSenderSchemaException.class, action::run); + Assert.assertEquals(error.getMessage(), reason, error.getReason()); + Assert.assertFalse(error.getMessage(), error.isRetryable()); + for (String part : messageParts) { + Assert.assertTrue(error.getMessage(), error.getMessage().contains(part)); + } + } + + static QwpSchemaBinding binding(QwpTableBuffer buffer, byte[]... columns) { + return binding(buffer, -1, columns); + } + + static QwpSchemaBinding binding(QwpTableBuffer buffer, int designatedIndex, byte[]... columns) { + return new QwpSchemaBinding(buffer, known(designatedIndex, columns)); + } + + static byte[] column(String name, int type) { + return column(name, type, NO_PARAMS); + } + + static byte[] column(String name, int type, byte[] params) { + byte[] bytes = name.getBytes(StandardCharsets.UTF_8); + return ByteBuffer.allocate(2 + bytes.length + 4 + 2 + params.length).order(ByteOrder.LITTLE_ENDIAN) + .putShort((short) bytes.length).put(bytes).putInt(type) + .putShort((short) params.length).put(params).array(); + } + + /** Wraps a control payload in a message frame and decodes it. */ + static QwpSchemaResponse decode(byte[] payload) { + byte[] frame = frame(payload); + long address = Unsafe.malloc(frame.length, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < frame.length; i++) { + Unsafe.getUnsafe().putByte(address + i, frame[i]); + } + return QwpSchemaProtocol.decodeResponse(address, frame.length); + } finally { + Unsafe.free(address, frame.length, MemoryTag.NATIVE_DEFAULT); + } + } + + static byte[] frame(byte[] payload) { + return ByteBuffer.allocate(QwpConstants.HEADER_SIZE + payload.length).order(ByteOrder.LITTLE_ENDIAN) + .putInt(QwpConstants.MAGIC_MESSAGE).put(QwpConstants.VERSION).put(QwpSchemaProtocol.FLAG_CONTROL) + .putShort((short) 0).putInt(payload.length).put(payload).array(); + } + + static QwpSchemaResponse known(byte[]... columns) { + return known(-1, columns); + } + + static QwpSchemaResponse known(int designatedIndex, byte[]... columns) { + int length = 1 + 8 + 1 + 4 + 8 + 2 + 2; + for (byte[] column : columns) { + length += column.length; + } + ByteBuffer payload = ByteBuffer.allocate(length).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) QwpSchemaProtocol.RESULT_KNOWN) + .putInt(1).putLong(1).putShort((short) designatedIndex).putShort((short) columns.length); + for (byte[] column : columns) { + payload.put(column); + } + return decode(payload.array()); + } + + static QwpSchemaResponse missing() { + return schemaResult(QwpSchemaProtocol.RESULT_MISSING); + } + + /** A column whose type carries a non-empty extension parameter block. */ + static byte[] parameterizedColumn(String name, int type) { + return column(name, type, new byte[]{1}); + } + + /** Decodes a schema-less response carrying only {@code result}. */ + static QwpSchemaResponse schemaResult(int result) { + return decode(ByteBuffer.allocate(10).order(ByteOrder.LITTLE_ENDIAN) + .put(QwpSchemaProtocol.KIND_SCHEMA).putLong(1).put((byte) result).array()); + } + + /** Consumes the table prefix and the header of a single column named "value". */ + static void skipTableHeader(QwpTestWireReader reader, byte wireType, int rows) { + skipTablePrefix(reader, rows, 1); + Assert.assertEquals("value", reader.string()); + Assert.assertEquals(wireType, reader.u8()); + } + + static void skipTablePrefix(QwpTestWireReader reader, int rows, int columns) { + skipTablePrefix(reader, rows, columns, 1, 1); + } + + /** Asserts the message header and table prefix, leaving the reader at the first column. */ + static QwpTestWireReader tableHeader(QwpWebSocketEncoder encoder, int size, int rows, int columns) { + return tableHeader(encoder, size, rows, columns, 1, 1); + } + + /** A negative {@code tableId} expects a schema-framed table block without a pinned identity. */ + static QwpTestWireReader tableHeader( + QwpWebSocketEncoder encoder, + int size, + int rows, + int columns, + int tableId, + long metadataVersion + ) { + QwpTestWireReader reader = new QwpTestWireReader(encoder.getBuffer().getBufferPtr(), size); + Assert.assertEquals(QwpConstants.MAGIC_MESSAGE, reader.i32()); + Assert.assertEquals(QwpConstants.VERSION, reader.u8()); + Assert.assertEquals(QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_SCHEMA, reader.u8()); + Assert.assertEquals(1, reader.u16()); + Assert.assertEquals(size - QwpConstants.HEADER_SIZE, reader.i32()); + skipTablePrefix(reader, rows, columns, tableId, metadataVersion); + return reader; + } + + private static void skipTablePrefix( + QwpTestWireReader reader, + int rows, + int columns, + int tableId, + long metadataVersion + ) { + Assert.assertEquals("t", reader.string()); + if (tableId < 0) { + Assert.assertEquals(0, reader.u8()); + } else { + Assert.assertEquals(1, reader.u8()); + Assert.assertEquals(tableId, reader.i32()); + Assert.assertEquals(metadataVersion, reader.i64()); + } + Assert.assertEquals(rows, reader.varint()); + Assert.assertEquals(columns, reader.varint()); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java index ab14c18d8..1be17770e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTableBufferTest.java @@ -1150,6 +1150,20 @@ public void testGetOrCreateColumnThrowsWhenExceedingMaxColumnCount() throws Exce }); } + @Test + public void testGetSchemaKeyLowerCasesOnlyWhenNeeded() throws Exception { + assertMemoryLeak(() -> { + String lowerCaseName = "trades"; + try (QwpTableBuffer table = new QwpTableBuffer(lowerCaseName)) { + assertSame(lowerCaseName, table.getSchemaKey()); + } + try (QwpTableBuffer table = new QwpTableBuffer("Trades_ÄB")) { + assertEquals("trades_äb", table.getSchemaKey()); + assertEquals("Trades_ÄB", table.getTableName()); + } + }); + } + @Test public void testLongArrayMultipleRows() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java new file mode 100644 index 000000000..94f2da358 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/protocol/QwpTestWireReader.java @@ -0,0 +1,140 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + ******************************************************************************/ +package io.questdb.client.test.cutlass.qwp.protocol; + +import io.questdb.client.std.Unsafe; +import org.junit.Assert; + +import java.nio.charset.StandardCharsets; + +/** + * Bounds-checked cursor over native memory holding an encoded QWP message. + * Tests decode the wire bytes with this reader rather than with production + * decoders, so an encoder bug cannot hide behind a matching decoder bug. + * Multi-byte reads use the native byte order, like the production encoder. + */ +final class QwpTestWireReader { + private final long address; + private final int limit; + private int position; + + QwpTestWireReader(long address, int limit) { + this.address = address; + this.limit = limit; + } + + /** Reads {@code length} raw bytes as a string, without a length prefix. */ + String ascii(int length) { + return new String(bytes(length), StandardCharsets.UTF_8); + } + + byte[] bytes(int length) { + require(length); + byte[] value = new byte[length]; + for (int i = 0; i < length; i++) { + value[i] = Unsafe.getUnsafe().getByte(address + position + i); + } + position += length; + return value; + } + + float f32() { + return Float.intBitsToFloat(i32()); + } + + double f64() { + return Double.longBitsToDouble(i64()); + } + + short i16() { + require(Short.BYTES); + short value = Unsafe.getUnsafe().getShort(address + position); + position += Short.BYTES; + return value; + } + + int i32() { + require(Integer.BYTES); + int value = Unsafe.getUnsafe().getInt(address + position); + position += Integer.BYTES; + return value; + } + + long i64() { + require(Long.BYTES); + long value = Unsafe.getUnsafe().getLong(address + position); + position += Long.BYTES; + return value; + } + + int limit() { + return limit; + } + + int position() { + return position; + } + + void skip(int length) { + require(length); + position += length; + } + + /** Reads a varint-prefixed UTF-8 string. */ + String string() { + return new String(stringBytes(), StandardCharsets.UTF_8); + } + + /** Reads the raw bytes of a varint-prefixed string. */ + byte[] stringBytes() { + return bytes(varint()); + } + + int u16() { + return i16() & 0xffff; + } + + int u8() { + require(1); + return Unsafe.getUnsafe().getByte(address + position++) & 0xff; + } + + int varint() { + int result = 0; + for (int shift = 0; shift < 32; shift += 7) { + int value = u8(); + result |= (value & 0x7f) << shift; + if ((value & 0x80) == 0) { + return result; + } + } + throw new AssertionError("varint exceeds 32 bits at position " + position); + } + + private void require(int length) { + Assert.assertTrue( + "read of " + length + " bytes at " + position + " exceeds limit " + limit, + length >= 0 && position <= limit - length + ); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 908e3bd56..8f0cf9120 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -125,6 +125,8 @@ public class TestWebSocketServer implements Closeable { // builds. Lets a test force the delta-dictionary catch-up to split across // several frames. Live-updatable via setAdvertisedMaxBatchSize(). private volatile int advertisedMaxBatchSize; + private volatile boolean advertiseSchema; + private volatile boolean hasRequestedSchema; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -190,6 +192,10 @@ public boolean awaitStart(long timeout, TimeUnit unit) throws InterruptedExcepti return startLatch.await(timeout, unit); } + public void setAdvertiseSchema(boolean advertiseSchema) { + this.advertiseSchema = advertiseSchema; + } + @Override public void close() { running.set(false); @@ -226,6 +232,10 @@ public int getPort() { return port; } + public boolean hasRequestedSchema() { + return hasRequestedSchema; + } + /** * Number of handshakes the server has completed over its lifetime * (monotonic; never decreases when clients disconnect). @@ -419,6 +429,12 @@ public void start() throws IOException { public interface WebSocketServerHandler { default void onBinaryMessage(ClientHandler client, byte[] data) { } + + default void onOpen(ClientHandler client) { + } + + default void onPong(ClientHandler client, byte[] data) { + } } /** @@ -463,6 +479,10 @@ public synchronized void sendBinary(byte[] data) throws IOException { writeFrame(WebSocketOpcode.BINARY, data, data.length); } + public synchronized void sendPing(byte[] data) throws IOException { + writeFrame(WebSocketOpcode.PING, data, data.length); + } + public synchronized void sendClose(int code, String reason) throws IOException { byte[] reasonBytes = (reason != null && !reason.isEmpty()) ? reason.getBytes(StandardCharsets.UTF_8) : new byte[0]; @@ -543,6 +563,9 @@ private void handleRead() { LOG.error("Failed to send pong", e); } break; + case WebSocketOpcode.PONG: + handler.onPong(this, payload); + break; case WebSocketOpcode.CLOSE: { int code = WebSocketCloseCode.NORMAL_CLOSURE; if (payload.length >= 2) { @@ -604,6 +627,8 @@ private boolean performHandshake() throws IOException { return false; } capturedAuthHeaders.add(authorization); + hasRequestedSchema |= request.toString().toLowerCase(java.util.Locale.ROOT) + .contains("x-qwp-request-schema: true\r\n"); // Read-path reject: drop the egress upgrade before the 101 so the // query pool's connect fails fast, while ingest write-path upgrades @@ -657,6 +682,9 @@ private boolean performHandshake() throws IOException { if (emitDurableAckHeader && !suppressDurableAckHeader) { sb.append("X-QWP-Durable-Ack: enabled\r\n"); } + if (advertiseSchema && request.toString().toLowerCase(java.util.Locale.ROOT).contains("x-qwp-request-schema: true\r\n")) { + sb.append("X-QWP-Schema: enabled\r\n"); + } String role = advertisedRole; if (role != null) { sb.append("X-QuestDB-Role: ").append(role).append("\r\n"); @@ -715,6 +743,7 @@ void start() { liveConnections.incrementAndGet(); try { + handler.onOpen(this); // SERVER_INFO is an egress-only frame: send it only on a // read-path (query) connection. An ingest write-path // connection parses every inbound frame as an ACK and diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index 3c257827a..9483edaf3 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -69,6 +69,8 @@ public void testEveryIngressKeyIsHonored() { assertHonored("close_flush_timeout_millis=2500", "close_flush_timeout_millis", 2500L); assertHonored("durable_ack_keepalive_interval_millis=900", "durable_ack_keepalive_interval_millis", 900L); assertHonored("initial_connect_retry=async", "initial_connect_retry", "ASYNC"); + assertHonored("schema_mode=strict", "schema_mode", "STRICT"); + assertHonored("schema_wait_millis=1234", "schema_wait_millis", 1234L); assertHonored("reconnect_max_duration_millis=12345", "reconnect_max_duration_millis", 12345L); assertHonored("reconnect_initial_backoff_millis=111", "reconnect_initial_backoff_millis", 111L); assertHonored("reconnect_max_backoff_millis=2222", "reconnect_max_backoff_millis", 2222L); diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java index 4a481e193..39b89680f 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal128Test.java @@ -891,7 +891,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal128 d = new Decimal128(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -928,7 +928,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -945,7 +945,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java index 1c461525d..a27d0e7ef 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal256Test.java @@ -1329,7 +1329,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal256 d = new Decimal256(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -1440,7 +1440,7 @@ public void testOfStringExponentComplexNumbers() throws NumericException { // Complex decimal with exponent int precision = Numbers.decodeLowInt(d.ofString("123.456789e-3")); - Assert.assertEquals(10, precision); + Assert.assertEquals(9, precision); Assert.assertEquals(9, d.getScale()); Assert.assertEquals("0.123456789", d.toString()); @@ -1463,7 +1463,7 @@ public void testOfStringExponentLargeButValid() throws NumericException { // Large negative exponent within bounds precision = Numbers.decodeLowInt(d.ofString("1e-20")); - Assert.assertEquals(21, precision); + Assert.assertEquals(20, precision); Assert.assertEquals(20, d.getScale()); Assert.assertEquals("0.00000000000000000001", d.toString()); } @@ -1530,7 +1530,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -1564,7 +1564,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } @@ -1581,7 +1581,7 @@ public void testOfStringExponentWithPrecisionLimit() throws NumericException { // With scale limit that matches the result scale precision = Numbers.decodeLowInt(d.ofString("1e-3", -1, 3)); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.001", d.toString()); } @@ -1612,7 +1612,7 @@ public void testOfStringExponentWithSuffix() throws NumericException { // With m suffix precision = Numbers.decodeLowInt(d.ofString("2.5e-3m")); - Assert.assertEquals(5, precision); + Assert.assertEquals(4, precision); Assert.assertEquals(4, d.getScale()); Assert.assertEquals("0.0025", d.toString()); } @@ -1832,7 +1832,7 @@ public void testOfStringPrecisionConsistency() throws NumericException { Assert.assertEquals(6, p3); int p4 = Numbers.decodeLowInt(d.ofString("00.001")); - Assert.assertEquals(4, p4); // Leading zeros don't count toward precision + Assert.assertEquals(3, p4); // Leading zeros don't count toward precision; 0.001 needs 3 fractional digits } @Test(expected = NumericException.class) @@ -1946,25 +1946,25 @@ public void testOfStringWithNegativeExponent() throws NumericException { // Basic negative exponent int precision = Numbers.decodeLowInt(d.ofString("1.234e-2")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.01234", d.toString()); // Larger negative exponent precision = Numbers.decodeLowInt(d.ofString("5e-5")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.00005", d.toString()); // Negative exponent with E (uppercase) precision = Numbers.decodeLowInt(d.ofString("7.89E-3")); - Assert.assertEquals(6, precision); + Assert.assertEquals(5, precision); Assert.assertEquals(5, d.getScale()); Assert.assertEquals("0.00789", d.toString()); // Very small number precision = Numbers.decodeLowInt(d.ofString("1e-10")); - Assert.assertEquals(11, precision); + Assert.assertEquals(10, precision); Assert.assertEquals(10, d.getScale()); Assert.assertEquals("0.0000000001", d.toString()); } @@ -2066,7 +2066,7 @@ public void testOfStringZeroWithDecimal() throws NumericException { Assert.assertEquals("0", d.toString()); precision = Numbers.decodeLowInt(d.ofString("0.000m")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.000", d.toString()); diff --git a/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java b/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java index 769d90120..008989f2a 100644 --- a/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java +++ b/core/src/test/java/io/questdb/client/test/std/Decimal64Test.java @@ -663,7 +663,7 @@ public void testOfStringDecimalPointAtEnd() throws NumericException { public void testOfStringDecimalPointAtStart() throws NumericException { Decimal64 d = new Decimal64(); int precision = Numbers.decodeLowInt(d.ofString(".123")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.123", d.toString()); } @@ -700,7 +700,7 @@ public void testOfStringExponentWithIntegerBase() throws NumericException { // Integer base with negative exponent precision = Numbers.decodeLowInt(d.ofString("456e-3")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("0.456", d.toString()); } @@ -717,7 +717,7 @@ public void testOfStringExponentWithNegativeNumber() throws NumericException { // Negative number with negative exponent precision = Numbers.decodeLowInt(d.ofString("-2.5e-2")); - Assert.assertEquals(4, precision); + Assert.assertEquals(3, precision); Assert.assertEquals(3, d.getScale()); Assert.assertEquals("-0.025", d.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java new file mode 100644 index 000000000..43ab94d9c --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/std/DecimalParserServerCompatibilityTest.java @@ -0,0 +1,119 @@ +/*+***************************************************************************** + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * Licensed under the Apache License, Version 2.0 + *******************************************************************************/ +package io.questdb.client.test.std; + +import io.questdb.client.std.Decimal256; +import io.questdb.client.std.DecimalParser; +import io.questdb.client.std.Numbers; +import io.questdb.client.std.NumericException; +import org.junit.Assert; +import org.junit.Test; + +public class DecimalParserServerCompatibilityTest { + @Test + public void testFullScaleAndTargetScalePaddingUseServerPrecisionRules() throws NumericException { + assertParsed("0.5", 1, 1, ".5", 1, 1); + assertParsed("0.500", 3, 3, "0.5", 3, 3); + assertParsed("0.100", 3, 3, ".1", 3, 3); + + String fraction = "0." + repeat('9', 76); + assertParsed(fraction, 76, 76, fraction, 76, 76); + } + + @Test + public void testLeadingZerosAndZeroExponentUseServerPrecisionRules() throws NumericException { + assertParsed("0", 1, 0, "0000.e80", 1, 0); + assertParsed("1", 1, 0, "0.01e2", -1, -1); + assertParsed("123", 3, 0, "00.123e3", -1, -1); + assertParsed("0", 1, 0, "0e5", -1, -1); + assertParsed("0.000", 3, 3, "0e-3", 3, 3); + } + + @Test + public void testPrecisionIsAtLeastOneAndZeroFitsFullScale() throws NumericException { + assertParsed("0", 1, 0, "0.000", -1, -1); + assertParsed("0", 1, 0, "0.", -1, -1); + assertParsed("0." + repeat('0', 76), 76, 76, "0", 76, 76); + } + + @Test + public void testSpecialValuesMatchByPrefixLikeServer() throws NumericException { + assertNullParsed("NaN", 38, 4); + assertNullParsed("-Infinity", 76, 8); + // The server's DecimalParser treats NaN and Infinity as prefixes. + assertNullParsed("NaNjunk", 38, 4); + assertNullParsed("NaN ", 38, 4); + assertNullParsed("Infinitygarbage", 76, 8); + assertRejected("Na", 38, 4); + assertRejected("Infinit", 76, 8); + } + + @Test + public void testSuffixOrderAndExtremeExponentsAreRejectedNormally() throws NumericException { + assertParsed("1.23", 3, 2, "1.2300", 38, 2); + assertRejected("1.2300M", 38, 2); + assertRejected("1.2300fm", 38, 2); + assertRejected("1.2300mf", 38, 2); + assertRejected("1e2147483647", 76, 0); + assertRejected("1e-2147483648", 76, 0); + } + + private static void assertParsed( + String expected, + int expectedPrecision, + int expectedScale, + String input, + int precision, + int scale + ) throws NumericException { + Decimal256 viaParser = new Decimal256(); + long parserMeta = DecimalParser.parse( + viaParser, input, 0, input.length(), precision, scale, false, false); + Assert.assertEquals(expected, viaParser.toString()); + Assert.assertEquals(expectedPrecision, Numbers.decodeLowInt(parserMeta)); + Assert.assertEquals(expectedScale, Numbers.decodeHighInt(parserMeta)); + + Decimal256 viaPublicDecimal = new Decimal256(); + long decimalMeta = viaPublicDecimal.ofString( + input, 0, input.length(), precision, scale, false, false); + Assert.assertEquals(expected, viaPublicDecimal.toString()); + Assert.assertEquals(expectedPrecision, Numbers.decodeLowInt(decimalMeta)); + Assert.assertEquals(expectedScale, Numbers.decodeHighInt(decimalMeta)); + } + + private static void assertNullParsed(String input, int precision, int scale) throws NumericException { + Decimal256 viaParser = new Decimal256(); + Assert.assertEquals(0, DecimalParser.parse( + viaParser, input, 0, input.length(), precision, scale, false, false)); + Assert.assertTrue(viaParser.isNull()); + + Decimal256 viaPublicDecimal = new Decimal256(); + Assert.assertEquals(0, viaPublicDecimal.ofString( + input, 0, input.length(), precision, scale, false, false)); + Assert.assertTrue(viaPublicDecimal.isNull()); + } + + private static void assertRejected(String input, int precision, int scale) { + try { + DecimalParser.parse(new Decimal256(), input, 0, input.length(), precision, scale, false, false); + Assert.fail("expected DecimalParser to reject: " + input); + } catch (NumericException expected) { + // expected + } + try { + new Decimal256().ofString(input, 0, input.length(), precision, scale, false, false); + Assert.fail("expected Decimal256 to reject: " + input); + } catch (NumericException expected) { + // expected + } + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + for (int i = 0; i < count; i++) chars[i] = value; + return new String(chars); + } +} diff --git a/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java b/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java index 9076319d3..2d2462a8a 100644 --- a/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java +++ b/core/src/test/java/io/questdb/client/test/std/DoubleFormatBenchmark.java @@ -49,18 +49,15 @@ * JMH benchmark for {@link Numbers#append(io.questdb.client.std.str.CharSink, double)}, * the double-to-text routine behind every double column the client sends. *

      - * Its bignum slow path goes through the {@code FdBig} bridge over the JDK's - * internal {@code FDBigInteger}. On Java 9+ that bridge binds the bignum - * methods via {@code static final} method handles resolved once at class-init - * (JDK 26 made the class package-private, so it can no longer be named in - * source). This benchmark exists to show the handle-based bridge costs the - * same as the direct calls it replaced: run it once against a client jar - * built from the previous {@code main} and once against the current tree, - * same JDK, and compare {@code shape=extreme} (100% slow path). + * {@code Numbers.append()} formats through Ryu (the same algorithm the server + * uses), which has no bignum slow path. The {@code shape} parameter still + * spans the input distributions that exercised the slow path of the pre-Ryu + * formatter, so the benchmark can compare the two across client versions: + * run it against a jar built from an older client and once against the + * current tree, same JDK. The slow-path shares below refer to that pre-Ryu + * formatter. *

      - * Input shapes, each a fixed pool of 1024 values cycled per invocation. The - * slow-path share was measured by running the pool against a pre-fix jar on - * JDK 26, where the bignum path throws and the fast path does not: + * Input shapes, each a fixed pool of 1024 values cycled per invocation: *

        *
      • {@code simple} -- short literals like {@code 1.0}, {@code 123456.789}; * 0/1024 slow path.
        • diff --git a/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java b/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java index 7bdb23a0a..64b7c2587 100644 --- a/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java +++ b/core/src/test/java/io/questdb/client/test/std/DoubleFormatSmoke.java @@ -29,7 +29,7 @@ /** * Standalone (no JUnit) smoke check that double formatting works against the * packaged jar on the running JDK. The extreme-exponent values exercise the - * FdBig fallback of {@code Numbers.appendDouble0}, which resolves + * Ryu double formatter, which resolves * {@code FDBigInteger} from a different JDK-internal package on Java 8 vs 9+; * a jar packaged with the wrong bridge dies here with * {@code NoClassDefFoundError: sun/misc/FDBigInteger} (the 1.3.5-1.3.7 @@ -43,7 +43,7 @@ public static void main(String[] args) { double[] values = { 0.0d, 123.456d, - // FdBig slow-path values + // formerly FdBig slow-path values 1.0E-300, Double.MIN_VALUE, Double.MAX_VALUE, diff --git a/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java b/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java index 8733fa150..890a7fa6f 100644 --- a/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java +++ b/core/src/test/java/io/questdb/client/test/std/JarPackagingIT.java @@ -43,14 +43,15 @@ *
      • the module name silently changing from {@code io.questdb.client} to the * filename-derived {@code questdb.client} (no module-info.class on JDK 8 * builds and no Automatic-Module-Name),
        • - *
      • JDK 8 builds shipping only the {@code sun.misc.FDBigInteger} FdBig - * bridge, which throws {@code NoClassDefFoundError} on Java 9+ without the - * {@code META-INF/versions/11} counterpart.
        • + *
      • JDK 8 builds shipping only the Java 8 {@code Compat} shim, whose + * {@code multiplyHigh} fallback the Ryu double formatter calls on every + * value, without the {@code META-INF/versions/11} counterpart that lets + * Java 9+ use the {@code Math.multiplyHigh} intrinsic instead.
        • *
      */ public class JarPackagingIT { - private static final String FD_BIG_ENTRY = "io/questdb/client/std/FdBig.class"; - private static final String VERSIONED_FD_BIG_ENTRY = "META-INF/versions/11/" + FD_BIG_ENTRY; + private static final String COMPAT_ENTRY = "io/questdb/client/std/Compat.class"; + private static final String VERSIONED_COMPAT_ENTRY = "META-INF/versions/11/" + COMPAT_ENTRY; @Test public void testDoubleFormattingAgainstPackagedJar() throws Exception { @@ -106,19 +107,16 @@ public void testJarLayout() throws Exception { // JDK 8 build: the shipping layout. Root classes target Java 8, // the java11 bridge rides in META-INF/versions/11. Assert.assertTrue( - "root FdBig of a JDK 8 build must use sun.misc.FDBigInteger", - classReferences(jar, FD_BIG_ENTRY, "sun/misc/FDBigInteger") - ); - // the java11 bridge never names the class (JDK 26 made it package-private); - // it loads it by dotted name and binds method handles, so the - // constant-pool witness is the dotted string literal - Assert.assertTrue( - "META-INF/versions/11 FdBig must bind jdk.internal.math.FDBigInteger", - classReferences(jar, VERSIONED_FD_BIG_ENTRY, "jdk.internal.math.FDBigInteger") + "root Compat of a JDK 8 build must use the Java 8 ManagementFactory variant", + classReferences(jar, COMPAT_ENTRY, "java/lang/management/ManagementFactory") ); Assert.assertNotNull( "META-INF/versions/11 must carry the java11 Compat shim", - jar.getEntry("META-INF/versions/11/io/questdb/client/std/Compat.class") + jar.getEntry(VERSIONED_COMPAT_ENTRY) + ); + Assert.assertTrue( + "META-INF/versions/11 Compat must use the Math.multiplyHigh intrinsic", + classReferences(jar, VERSIONED_COMPAT_ENTRY, "multiplyHigh") ); // every source in src/main/java11 must ship in versions/11 -- a file added // to the source root but missed by the packaging step would recreate the @@ -133,8 +131,8 @@ public void testJarLayout() throws Exception { // JDK 11+ build: dev/smoke only, never shipped. Root classes are the // java11 variants and the real module descriptor is present. Assert.assertTrue( - "root FdBig of a JDK 11+ build must bind jdk.internal.math.FDBigInteger", - classReferences(jar, FD_BIG_ENTRY, "jdk.internal.math.FDBigInteger") + "root Compat of a JDK 11+ build must use the Math.multiplyHigh intrinsic", + classReferences(jar, COMPAT_ENTRY, "multiplyHigh") ); Assert.assertNotNull("module-info.class missing", jar.getEntry("module-info.class")); } diff --git a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java index 0467235da..5b5c6c15b 100644 --- a/core/src/test/java/io/questdb/client/test/std/NumbersTest.java +++ b/core/src/test/java/io/questdb/client/test/std/NumbersTest.java @@ -140,7 +140,10 @@ public void testFormatDoubleExpNeg() { public void testFormatDoubleFast() { sink.clear(); Numbers.append(sink, -5.9522650387500933e18); - TestUtils.assertEquals("-5.9522650387500933E18", sink); + // Shortest form that still reads back to the same double. The previous + // formatter emitted a 17th digit ("...0933E18"); both parse to + // -0x1.4a6ac610aeb44p62, and Double.toString agrees with this one. + TestUtils.assertEquals("-5.952265038750093E18", sink); } @Test @@ -536,6 +539,16 @@ public void testParseIPv4LeadingDots3() { Numbers.parseIPv4("...a.1.2.3.4"); } + @Test(expected = NumericException.class) + public void testParseIPv4OnlyDots() { + Numbers.parseIPv4("...."); + } + + @Test(expected = NumericException.class) + public void testParseIPv4SingleDot() { + Numbers.parseIPv4("."); + } + @Test(expected = NumericException.class) public void testParseIPv4MiddleDots() { Numbers.parseIPv4("1..2..3..4"); diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv new file mode 100644 index 000000000..21a5afe70 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/binary-input.tsv @@ -0,0 +1,21 @@ +# case_id input_kind input_data_hex expected_wire_type expected_wire_hex expected_sql_hex outcome +# BINARY input is raw bytes; STRING input is UTF-16BE code units. Empty is a present zero-length value. +# is an explicit bitmap null. Hex is lowercase and separator-free. +binary_empty BINARY BINARY VALID +binary_zero BINARY 00 BINARY 00 00 VALID +binary_opaque BINARY 00ff807f010203 BINARY 00ff807f010203 00ff807f010203 VALID +binary_all_256 BINARY 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff BINARY 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff VALID +string_null STRING BINARY VALID +string_empty STRING BINARY VALID +string_ascii STRING 00510075006500730074 BINARY 5175657374 5175657374 VALID +string_embedded_nul STRING 006100000062 BINARY 610062 610062 VALID +string_bmp_two_byte STRING 00e9 BINARY c3a9 c3a9 VALID +string_bmp_three_byte STRING 6c49 BINARY e6b189 e6b189 VALID +string_supplementary STRING d83dde00 BINARY f09f9880 f09f9880 VALID +string_lone_high STRING d800 BINARY 3f 3f VALID +string_high_ascii STRING d8000061 BINARY 3f61 3f61 VALID +string_high_high_low STRING d800d83dde00 BINARY 3ff09f9880 3ff09f9880 VALID +string_lone_low STRING dc00 BINARY 3f 3f VALID +string_low_high_ascii STRING dc00d8000061 BINARY 3f3f61 3f3f61 VALID +string_literal_question STRING 003f BINARY 3f 3f VALID +string_controls STRING 00610009000d000a0062 BINARY 61090d0a62 61090d0a62 VALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv new file mode 100644 index 000000000..8bcc6a231 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/boolean-to-target.tsv @@ -0,0 +1,19 @@ +# case_id input_boolean target_type expected_wire_type expected_wire_value expected_sql_value +bool_false_boolean FALSE BOOLEAN BOOLEAN 0 false +bool_true_boolean TRUE BOOLEAN BOOLEAN 1 true +bool_false_byte FALSE BYTE BYTE 0 0 +bool_true_byte TRUE BYTE BYTE 1 1 +bool_false_short FALSE SHORT SHORT 0 0 +bool_true_short TRUE SHORT SHORT 1 1 +bool_false_int FALSE INT INT 0 0 +bool_true_int TRUE INT INT 1 1 +bool_false_long FALSE LONG LONG 0 0 +bool_true_long TRUE LONG LONG 1 1 +bool_false_float FALSE FLOAT FLOAT 0 0 +bool_true_float TRUE FLOAT FLOAT 1 1 +bool_false_double FALSE DOUBLE DOUBLE 0 0 +bool_true_double TRUE DOUBLE DOUBLE 1 1 +bool_false_string FALSE STRING VARCHAR 66616c7365 00660061006c00730065 +bool_true_string TRUE STRING VARCHAR 74727565 0074007200750065 +bool_false_varchar FALSE VARCHAR VARCHAR 66616c7365 00660061006c00730065 +bool_true_varchar TRUE VARCHAR VARCHAR 74727565 0074007200750065 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv new file mode 100644 index 000000000..cfb464ac3 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/char-conversions.tsv @@ -0,0 +1,24 @@ +# case_id input_kind utf16_hex expected_char expected_null +char_nul CHAR 0000 0 true +char_ascii CHAR 0041 65 false +char_two_byte CHAR 00e9 233 false +char_three_byte CHAR 20ac 8364 false +char_before_surrogate CHAR d7ff 55295 false +char_high_surrogate CHAR d83d 55357 false +char_low_surrogate CHAR de00 56832 false +char_after_surrogate CHAR e000 57344 false +char_max CHAR ffff 65535 false +string_null STRING 0 true +string_empty STRING 0 true +string_nul STRING 0000 0 true +string_ascii STRING 0041 65 false +string_two_byte STRING 00e9 233 false +string_three_byte STRING 20ac 8364 false +string_before_surrogate STRING d7ff 55295 false +string_after_surrogate STRING e000 57344 false +string_max STRING ffff 65535 false +string_supplementary_first STRING d83dde00 0 true +string_lone_high STRING d83d 63 false +string_lone_low STRING de00 63 false +string_high_then_ascii STRING d83d0041 63 false +string_ascii_then_trailing STRING 004200e9d83dde00 66 false diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv new file mode 100644 index 000000000..f6468b645 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-decimal.tsv @@ -0,0 +1,61 @@ +# case_id source_type input_bits_hex target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +float_to_decimal8 FLOAT 41180000 DECIMAL8 2 1 VALUE 000000000000005f - - - 9.5 +double_to_decimal8 DOUBLE 3fb999999999999a DECIMAL8 2 1 VALUE 0000000000000001 - - - 0.1 +float_to_decimal16 FLOAT c1480000 DECIMAL16 4 2 VALUE fffffffffffffb1e - - - -12.50 +double_to_decimal16 DOUBLE 4058ff5c28f5c28f DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +float_to_decimal32 FLOAT 4640e480 DECIMAL32 9 3 VALUE 0000000000bc5f25 - - - 12345.125 +double_to_decimal32 DOUBLE c08f3ffdf3b645a2 DECIMAL32 9 3 VALUE fffffffffff0bdc1 - - - -999.999 +float_to_decimal64 FLOAT 3dcccccd DECIMAL64 18 17 VALUE 002386f278a2bc9c - - - 0.10000000149011612 +double_to_decimal64 DOUBLE 3fb999999999999a DECIMAL64 18 17 VALUE 002386f26fc10000 - - - 0.10000000000000000 +float_to_decimal128 FLOAT 4b800000 DECIMAL128 24 5 VALUE 00000186a0000000 0000000000000000 - - 16777216.00000 +double_to_decimal128 DOUBLE 44b52d02c7e14af6 DECIMAL128 24 0 VALUE 02c7e14af6800000 000000000000152d - - 100000000000000000000000 +float_to_decimal256 FLOAT 7f7fffff DECIMAL256 76 0 VALUE 35211f7ca1800000 ffffff000000000a 0000000000000000 0000000000000000 340282346638528860000000000000000000000 +double_to_decimal256 DOUBLE 3027288e1271f513 DECIMAL256 76 76 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +decimal8_positive_limit DOUBLE 4023cccccccccccd DECIMAL8 2 1 VALUE 0000000000000063 - - - 9.9 +decimal8_negative_limit DOUBLE c023cccccccccccd DECIMAL8 2 1 VALUE ffffffffffffff9d - - - -9.9 +decimal8_positive_over FLOAT 41200000 DECIMAL8 2 1 INVALID - - - - - +decimal8_negative_over DOUBLE c024000000000000 DECIMAL8 2 1 INVALID - - - - - +decimal16_positive_limit DOUBLE 4058ff5c28f5c28f DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +decimal16_negative_fit FLOAT c2c70000 DECIMAL16 4 2 VALUE ffffffffffffd922 - - - -99.50 +decimal16_positive_over DOUBLE 4059000000000000 DECIMAL16 4 2 INVALID - - - - - +decimal16_negative_over FLOAT c2c80000 DECIMAL16 4 2 INVALID - - - - - +decimal32_positive_limit DOUBLE 412e847fff7ced91 DECIMAL32 9 3 VALUE 000000003b9ac9ff - - - 999999.999 +decimal32_positive_fit FLOAT 497423f8 DECIMAL32 9 3 VALUE 000000003b9ac80c - - - 999999.500 +decimal32_positive_over DOUBLE 412e848000000000 DECIMAL32 9 3 INVALID - - - - - +decimal32_negative_over FLOAT c9742400 DECIMAL32 9 3 INVALID - - - - - +decimal64_positive_fit DOUBLE 4376345785d89fff DECIMAL64 18 1 VALUE 0de0b6b3a763ff38 - - - 99999999999999980.0 +decimal64_positive_over DOUBLE 4376345785d8a000 DECIMAL64 18 1 INVALID - - - - - +decimal128_positive_fit DOUBLE 43abc16d674ec7ff DECIMAL128 38 20 VALUE efa958854dc00000 4b3b4ca85a86c25b - - 999999999999999900.00000000000000000000 +decimal128_positive_over DOUBLE 43abc16d674ec800 DECIMAL128 38 20 INVALID - - - - - +decimal256_positive_fit DOUBLE 43e158e460913cff DECIMAL256 76 57 VALUE e000000000000000 c8b1219848c9ee4d 6916687cb4209f75 161bcca711991476 9999999999999998000.000000000000000000000000000000000000000000000000000000000 +decimal256_positive_over DOUBLE 43e158e460913d00 DECIMAL256 76 57 INVALID - - - - - +double_no_rounding DOUBLE 3ff4000000000000 DECIMAL8 2 1 INVALID - - - - - +float_no_rounding FLOAT 3fa00000 DECIMAL8 2 1 INVALID - - - - - +widened_float_point_one_scale_reject FLOAT 3dcccccd DECIMAL8 2 1 INVALID - - - - - +double_point_one_scale_accept DOUBLE 3fb999999999999a DECIMAL8 2 1 VALUE 0000000000000001 - - - 0.1 +float_positive_zero FLOAT 00000000 DECIMAL16 4 2 VALUE 0000000000000000 - - - 0.00 +float_negative_zero FLOAT 80000000 DECIMAL32 9 3 VALUE 0000000000000000 - - - 0.000 +double_positive_zero DOUBLE 0000000000000000 DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +double_negative_zero DOUBLE 8000000000000000 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +float_nan_payload FLOAT 7fc00001 DECIMAL8 2 1 NULL - - - - NULL +float_positive_infinity FLOAT 7f800000 DECIMAL16 4 2 NULL - - - - NULL +float_negative_infinity FLOAT ff800000 DECIMAL32 9 3 NULL - - - - NULL +double_nan_payload DOUBLE 7ff8000000000042 DECIMAL64 18 8 NULL - - - - NULL +double_positive_infinity DOUBLE 7ff0000000000000 DECIMAL128 38 20 NULL - - - - NULL +double_negative_infinity DOUBLE fff0000000000000 DECIMAL256 76 57 NULL - - - - NULL +float_min_value_widened_scale_60 FLOAT 00000001 DECIMAL256 60 60 VALUE 0004fa793930bcd1 0000000000000000 0000000000000000 0000000000000000 0.000000000000000000000000000000000000000000001401298464324817 +float_min_value_widened_scale_59_reject FLOAT 00000001 DECIMAL256 59 59 INVALID - - - - - +float_max_value FLOAT 7f7fffff DECIMAL256 76 0 VALUE 35211f7ca1800000 ffffff000000000a 0000000000000000 0000000000000000 340282346638528860000000000000000000000 +double_one_e_minus_76 DOUBLE 3027288e1271f513 DECIMAL256 76 76 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +double_one_e_minus_77_reject DOUBLE 2ff286d80ec190dc DECIMAL256 76 76 INVALID - - - - - +double_min_normal_reject DOUBLE 0010000000000000 DECIMAL256 76 76 INVALID - - - - - +double_min_value_reject DOUBLE 0000000000000001 DECIMAL256 76 76 INVALID - - - - - +double_max_value_reject DOUBLE 7fefffffffffffff DECIMAL256 76 0 INVALID - - - - - +float_below_two_pow_24 FLOAT 4b7fffff DECIMAL64 18 0 VALUE 0000000000ffffff - - - 16777215 +float_two_pow_24 FLOAT 4b800000 DECIMAL64 18 0 VALUE 0000000001000000 - - - 16777216 +float_above_two_pow_24 FLOAT 4b800001 DECIMAL64 18 0 VALUE 0000000001000002 - - - 16777218 +double_below_two_pow_53 DOUBLE 433fffffffffffff DECIMAL64 18 0 VALUE 001fffffffffffff - - - 9007199254740991 +double_two_pow_53 DOUBLE 4340000000000000 DECIMAL64 18 0 VALUE 0020000000000000 - - - 9007199254740992 +double_above_two_pow_53 DOUBLE 4340000000000001 DECIMAL64 18 0 VALUE 0020000000000002 - - - 9007199254740994 +double_repeating_shortest DOUBLE 3fd5555555555555 DECIMAL64 18 17 VALUE 00766c7d74835552 - - - 0.33333333333333330 +double_large_ryu_boundary DOUBLE 44b52d02c7e14af6 DECIMAL128 24 0 VALUE 02c7e14af6800000 000000000000152d - - 100000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv new file mode 100644 index 000000000..6c5d59e36 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-numeric.tsv @@ -0,0 +1,353 @@ +# Seven fields: case_id, input_type, input_raw_bits, target_type, expected_wire_type, expected_wire_bits, expected_sql_value +# Raw bits are lowercase fixed-width hexadecimal. is bitmap null/no value bytes; paired fields are local rejection. +# BITS: means stored floating raw bits. Present INT/LONG minimum uses numeric wire bits and SQL NULL. +float_byte_pos_zero FLOAT 00000000 BYTE BYTE 00 0 +float_byte_neg_zero FLOAT 80000000 BYTE BYTE 00 0 +float_byte_pos_one FLOAT 3f800000 BYTE BYTE 01 1 +float_byte_neg_one FLOAT bf800000 BYTE BYTE ff -1 +float_byte_fraction FLOAT 3fc00000 BYTE BYTE +float_byte_pos_inf FLOAT 7f800000 BYTE BYTE +float_byte_neg_inf FLOAT ff800000 BYTE BYTE +float_byte_nan_payload FLOAT 7fc00001 BYTE BYTE 0 +float_byte_nan_negative FLOAT ffc12345 BYTE BYTE 0 +float_byte_min_subnormal FLOAT 00000001 BYTE BYTE +float_byte_max_finite FLOAT 7f7fffff BYTE BYTE +float_byte_byte_neg_128 FLOAT c3000000 BYTE BYTE 80 -128 +float_byte_byte_pos_127 FLOAT 42fe0000 BYTE BYTE 7f 127 +float_byte_byte_neg_129 FLOAT c3010000 BYTE BYTE +float_byte_byte_pos_128 FLOAT 43000000 BYTE BYTE +float_byte_short_neg_32768 FLOAT c7000000 BYTE BYTE +float_byte_short_pos_32767 FLOAT 46fffe00 BYTE BYTE +float_byte_short_neg_32769 FLOAT c7000100 BYTE BYTE +float_byte_short_pos_32768 FLOAT 47000000 BYTE BYTE +float_byte_int_neg_2147483648 FLOAT cf000000 BYTE BYTE +float_byte_int_pos_2147483647 FLOAT 4f000000 BYTE BYTE +float_byte_int_neg_2147483649 FLOAT cf000000 BYTE BYTE +float_byte_int_pos_2147483648 FLOAT 4f000000 BYTE BYTE +float_byte_long_neg_9223372036854776000 FLOAT df000000 BYTE BYTE +float_byte_long_pos_9223372036854776000 FLOAT 5f000000 BYTE BYTE +float_short_pos_zero FLOAT 00000000 SHORT SHORT 0000 0 +float_short_neg_zero FLOAT 80000000 SHORT SHORT 0000 0 +float_short_pos_one FLOAT 3f800000 SHORT SHORT 0001 1 +float_short_neg_one FLOAT bf800000 SHORT SHORT ffff -1 +float_short_fraction FLOAT 3fc00000 SHORT SHORT +float_short_pos_inf FLOAT 7f800000 SHORT SHORT +float_short_neg_inf FLOAT ff800000 SHORT SHORT +float_short_nan_payload FLOAT 7fc00001 SHORT SHORT 0 +float_short_nan_negative FLOAT ffc12345 SHORT SHORT 0 +float_short_min_subnormal FLOAT 00000001 SHORT SHORT +float_short_max_finite FLOAT 7f7fffff SHORT SHORT +float_short_byte_neg_128 FLOAT c3000000 SHORT SHORT ff80 -128 +float_short_byte_pos_127 FLOAT 42fe0000 SHORT SHORT 007f 127 +float_short_byte_neg_129 FLOAT c3010000 SHORT SHORT ff7f -129 +float_short_byte_pos_128 FLOAT 43000000 SHORT SHORT 0080 128 +float_short_short_neg_32768 FLOAT c7000000 SHORT SHORT 8000 -32768 +float_short_short_pos_32767 FLOAT 46fffe00 SHORT SHORT 7fff 32767 +float_short_short_neg_32769 FLOAT c7000100 SHORT SHORT +float_short_short_pos_32768 FLOAT 47000000 SHORT SHORT +float_short_int_neg_2147483648 FLOAT cf000000 SHORT SHORT +float_short_int_pos_2147483647 FLOAT 4f000000 SHORT SHORT +float_short_int_neg_2147483649 FLOAT cf000000 SHORT SHORT +float_short_int_pos_2147483648 FLOAT 4f000000 SHORT SHORT +float_short_long_neg_9223372036854776000 FLOAT df000000 SHORT SHORT +float_short_long_pos_9223372036854776000 FLOAT 5f000000 SHORT SHORT +float_int_pos_zero FLOAT 00000000 INT INT 00000000 0 +float_int_neg_zero FLOAT 80000000 INT INT 00000000 0 +float_int_pos_one FLOAT 3f800000 INT INT 00000001 1 +float_int_neg_one FLOAT bf800000 INT INT ffffffff -1 +float_int_fraction FLOAT 3fc00000 INT INT +float_int_pos_inf FLOAT 7f800000 INT INT +float_int_neg_inf FLOAT ff800000 INT INT +float_int_nan_payload FLOAT 7fc00001 INT INT NULL +float_int_nan_negative FLOAT ffc12345 INT INT NULL +float_int_min_subnormal FLOAT 00000001 INT INT +float_int_max_finite FLOAT 7f7fffff INT INT +float_int_byte_neg_128 FLOAT c3000000 INT INT ffffff80 -128 +float_int_byte_pos_127 FLOAT 42fe0000 INT INT 0000007f 127 +float_int_byte_neg_129 FLOAT c3010000 INT INT ffffff7f -129 +float_int_byte_pos_128 FLOAT 43000000 INT INT 00000080 128 +float_int_short_neg_32768 FLOAT c7000000 INT INT ffff8000 -32768 +float_int_short_pos_32767 FLOAT 46fffe00 INT INT 00007fff 32767 +float_int_short_neg_32769 FLOAT c7000100 INT INT ffff7fff -32769 +float_int_short_pos_32768 FLOAT 47000000 INT INT 00008000 32768 +float_int_int_neg_2147483648 FLOAT cf000000 INT INT 80000000 NULL +float_int_int_pos_2147483647 FLOAT 4f000000 INT INT +float_int_int_neg_2147483649 FLOAT cf000000 INT INT 80000000 NULL +float_int_int_pos_2147483648 FLOAT 4f000000 INT INT +float_int_long_neg_9223372036854776000 FLOAT df000000 INT INT +float_int_long_pos_9223372036854776000 FLOAT 5f000000 INT INT +float_long_pos_zero FLOAT 00000000 LONG LONG 0000000000000000 0 +float_long_neg_zero FLOAT 80000000 LONG LONG 0000000000000000 0 +float_long_pos_one FLOAT 3f800000 LONG LONG 0000000000000001 1 +float_long_neg_one FLOAT bf800000 LONG LONG ffffffffffffffff -1 +float_long_fraction FLOAT 3fc00000 LONG LONG +float_long_pos_inf FLOAT 7f800000 LONG LONG +float_long_neg_inf FLOAT ff800000 LONG LONG +float_long_nan_payload FLOAT 7fc00001 LONG LONG NULL +float_long_nan_negative FLOAT ffc12345 LONG LONG NULL +float_long_min_subnormal FLOAT 00000001 LONG LONG +float_long_max_finite FLOAT 7f7fffff LONG LONG +float_long_byte_neg_128 FLOAT c3000000 LONG LONG ffffffffffffff80 -128 +float_long_byte_pos_127 FLOAT 42fe0000 LONG LONG 000000000000007f 127 +float_long_byte_neg_129 FLOAT c3010000 LONG LONG ffffffffffffff7f -129 +float_long_byte_pos_128 FLOAT 43000000 LONG LONG 0000000000000080 128 +float_long_short_neg_32768 FLOAT c7000000 LONG LONG ffffffffffff8000 -32768 +float_long_short_pos_32767 FLOAT 46fffe00 LONG LONG 0000000000007fff 32767 +float_long_short_neg_32769 FLOAT c7000100 LONG LONG ffffffffffff7fff -32769 +float_long_short_pos_32768 FLOAT 47000000 LONG LONG 0000000000008000 32768 +float_long_int_neg_2147483648 FLOAT cf000000 LONG LONG ffffffff80000000 -2147483648 +float_long_int_pos_2147483647 FLOAT 4f000000 LONG LONG 0000000080000000 2147483648 +float_long_int_neg_2147483649 FLOAT cf000000 LONG LONG ffffffff80000000 -2147483648 +float_long_int_pos_2147483648 FLOAT 4f000000 LONG LONG 0000000080000000 2147483648 +float_long_long_neg_9223372036854776000 FLOAT df000000 LONG LONG 8000000000000000 NULL +float_long_long_pos_9223372036854776000 FLOAT 5f000000 LONG LONG +float_float_pos_zero FLOAT 00000000 FLOAT FLOAT 00000000 BITS:00000000 +float_float_neg_zero FLOAT 80000000 FLOAT FLOAT 80000000 BITS:80000000 +float_float_pos_one FLOAT 3f800000 FLOAT FLOAT 3f800000 BITS:3f800000 +float_float_neg_one FLOAT bf800000 FLOAT FLOAT bf800000 BITS:bf800000 +float_float_fraction FLOAT 3fc00000 FLOAT FLOAT 3fc00000 BITS:3fc00000 +float_float_pos_inf FLOAT 7f800000 FLOAT FLOAT 7f800000 BITS:7f800000 +float_float_neg_inf FLOAT ff800000 FLOAT FLOAT ff800000 BITS:ff800000 +float_float_nan_payload FLOAT 7fc00001 FLOAT FLOAT NULL +float_float_nan_negative FLOAT ffc12345 FLOAT FLOAT NULL +float_float_min_subnormal FLOAT 00000001 FLOAT FLOAT 00000001 BITS:00000001 +float_float_max_finite FLOAT 7f7fffff FLOAT FLOAT 7f7fffff BITS:7f7fffff +float_float_byte_neg_128 FLOAT c3000000 FLOAT FLOAT c3000000 BITS:c3000000 +float_float_byte_pos_127 FLOAT 42fe0000 FLOAT FLOAT 42fe0000 BITS:42fe0000 +float_float_byte_neg_129 FLOAT c3010000 FLOAT FLOAT c3010000 BITS:c3010000 +float_float_byte_pos_128 FLOAT 43000000 FLOAT FLOAT 43000000 BITS:43000000 +float_float_short_neg_32768 FLOAT c7000000 FLOAT FLOAT c7000000 BITS:c7000000 +float_float_short_pos_32767 FLOAT 46fffe00 FLOAT FLOAT 46fffe00 BITS:46fffe00 +float_float_short_neg_32769 FLOAT c7000100 FLOAT FLOAT c7000100 BITS:c7000100 +float_float_short_pos_32768 FLOAT 47000000 FLOAT FLOAT 47000000 BITS:47000000 +float_float_int_neg_2147483648 FLOAT cf000000 FLOAT FLOAT cf000000 BITS:cf000000 +float_float_int_pos_2147483647 FLOAT 4f000000 FLOAT FLOAT 4f000000 BITS:4f000000 +float_float_int_neg_2147483649 FLOAT cf000000 FLOAT FLOAT cf000000 BITS:cf000000 +float_float_int_pos_2147483648 FLOAT 4f000000 FLOAT FLOAT 4f000000 BITS:4f000000 +float_float_long_neg_9223372036854776000 FLOAT df000000 FLOAT FLOAT df000000 BITS:df000000 +float_float_long_pos_9223372036854776000 FLOAT 5f000000 FLOAT FLOAT 5f000000 BITS:5f000000 +float_double_pos_zero FLOAT 00000000 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 +float_double_neg_zero FLOAT 80000000 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 +float_double_pos_one FLOAT 3f800000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 +float_double_neg_one FLOAT bf800000 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 +float_double_fraction FLOAT 3fc00000 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 +float_double_pos_inf FLOAT 7f800000 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 +float_double_neg_inf FLOAT ff800000 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 +float_double_nan_payload FLOAT 7fc00001 DOUBLE DOUBLE NULL +float_double_nan_negative FLOAT ffc12345 DOUBLE DOUBLE NULL +float_double_min_subnormal FLOAT 00000001 DOUBLE DOUBLE 36a0000000000000 BITS:36a0000000000000 +float_double_max_finite FLOAT 7f7fffff DOUBLE DOUBLE 47efffffe0000000 BITS:47efffffe0000000 +float_double_byte_neg_128 FLOAT c3000000 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 +float_double_byte_pos_127 FLOAT 42fe0000 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 +float_double_byte_neg_129 FLOAT c3010000 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 +float_double_byte_pos_128 FLOAT 43000000 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 +float_double_short_neg_32768 FLOAT c7000000 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 +float_double_short_pos_32767 FLOAT 46fffe00 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 +float_double_short_neg_32769 FLOAT c7000100 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 +float_double_short_pos_32768 FLOAT 47000000 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 +float_double_int_neg_2147483648 FLOAT cf000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +float_double_int_pos_2147483647 FLOAT 4f000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +float_double_int_neg_2147483649 FLOAT cf000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +float_double_int_pos_2147483648 FLOAT 4f000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +float_double_long_neg_9223372036854776000 FLOAT df000000 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 +float_double_long_pos_9223372036854776000 FLOAT 5f000000 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 +double_byte_pos_zero DOUBLE 0000000000000000 BYTE BYTE 00 0 +double_byte_neg_zero DOUBLE 8000000000000000 BYTE BYTE 00 0 +double_byte_pos_one DOUBLE 3ff0000000000000 BYTE BYTE 01 1 +double_byte_neg_one DOUBLE bff0000000000000 BYTE BYTE ff -1 +double_byte_fraction DOUBLE 3ff8000000000000 BYTE BYTE +double_byte_pos_inf DOUBLE 7ff0000000000000 BYTE BYTE +double_byte_neg_inf DOUBLE fff0000000000000 BYTE BYTE +double_byte_nan_payload DOUBLE 7ff8000000000001 BYTE BYTE 0 +double_byte_nan_negative DOUBLE fff8000000000042 BYTE BYTE 0 +double_byte_min_subnormal DOUBLE 0000000000000001 BYTE BYTE +double_byte_max_finite DOUBLE 7fefffffffffffff BYTE BYTE +double_byte_byte_neg_128 DOUBLE c060000000000000 BYTE BYTE 80 -128 +double_byte_byte_pos_127 DOUBLE 405fc00000000000 BYTE BYTE 7f 127 +double_byte_byte_neg_129 DOUBLE c060200000000000 BYTE BYTE +double_byte_byte_pos_128 DOUBLE 4060000000000000 BYTE BYTE +double_byte_short_neg_32768 DOUBLE c0e0000000000000 BYTE BYTE +double_byte_short_pos_32767 DOUBLE 40dfffc000000000 BYTE BYTE +double_byte_short_neg_32769 DOUBLE c0e0002000000000 BYTE BYTE +double_byte_short_pos_32768 DOUBLE 40e0000000000000 BYTE BYTE +double_byte_int_neg_2147483648 DOUBLE c1e0000000000000 BYTE BYTE +double_byte_int_pos_2147483647 DOUBLE 41dfffffffc00000 BYTE BYTE +double_byte_int_neg_2147483649 DOUBLE c1e0000000200000 BYTE BYTE +double_byte_int_pos_2147483648 DOUBLE 41e0000000000000 BYTE BYTE +double_byte_long_neg_9223372036854776000 DOUBLE c3e0000000000000 BYTE BYTE +double_byte_long_pos_9223372036854776000 DOUBLE 43e0000000000000 BYTE BYTE +double_short_pos_zero DOUBLE 0000000000000000 SHORT SHORT 0000 0 +double_short_neg_zero DOUBLE 8000000000000000 SHORT SHORT 0000 0 +double_short_pos_one DOUBLE 3ff0000000000000 SHORT SHORT 0001 1 +double_short_neg_one DOUBLE bff0000000000000 SHORT SHORT ffff -1 +double_short_fraction DOUBLE 3ff8000000000000 SHORT SHORT +double_short_pos_inf DOUBLE 7ff0000000000000 SHORT SHORT +double_short_neg_inf DOUBLE fff0000000000000 SHORT SHORT +double_short_nan_payload DOUBLE 7ff8000000000001 SHORT SHORT 0 +double_short_nan_negative DOUBLE fff8000000000042 SHORT SHORT 0 +double_short_min_subnormal DOUBLE 0000000000000001 SHORT SHORT +double_short_max_finite DOUBLE 7fefffffffffffff SHORT SHORT +double_short_byte_neg_128 DOUBLE c060000000000000 SHORT SHORT ff80 -128 +double_short_byte_pos_127 DOUBLE 405fc00000000000 SHORT SHORT 007f 127 +double_short_byte_neg_129 DOUBLE c060200000000000 SHORT SHORT ff7f -129 +double_short_byte_pos_128 DOUBLE 4060000000000000 SHORT SHORT 0080 128 +double_short_short_neg_32768 DOUBLE c0e0000000000000 SHORT SHORT 8000 -32768 +double_short_short_pos_32767 DOUBLE 40dfffc000000000 SHORT SHORT 7fff 32767 +double_short_short_neg_32769 DOUBLE c0e0002000000000 SHORT SHORT +double_short_short_pos_32768 DOUBLE 40e0000000000000 SHORT SHORT +double_short_int_neg_2147483648 DOUBLE c1e0000000000000 SHORT SHORT +double_short_int_pos_2147483647 DOUBLE 41dfffffffc00000 SHORT SHORT +double_short_int_neg_2147483649 DOUBLE c1e0000000200000 SHORT SHORT +double_short_int_pos_2147483648 DOUBLE 41e0000000000000 SHORT SHORT +double_short_long_neg_9223372036854776000 DOUBLE c3e0000000000000 SHORT SHORT +double_short_long_pos_9223372036854776000 DOUBLE 43e0000000000000 SHORT SHORT +double_int_pos_zero DOUBLE 0000000000000000 INT INT 00000000 0 +double_int_neg_zero DOUBLE 8000000000000000 INT INT 00000000 0 +double_int_pos_one DOUBLE 3ff0000000000000 INT INT 00000001 1 +double_int_neg_one DOUBLE bff0000000000000 INT INT ffffffff -1 +double_int_fraction DOUBLE 3ff8000000000000 INT INT +double_int_pos_inf DOUBLE 7ff0000000000000 INT INT +double_int_neg_inf DOUBLE fff0000000000000 INT INT +double_int_nan_payload DOUBLE 7ff8000000000001 INT INT NULL +double_int_nan_negative DOUBLE fff8000000000042 INT INT NULL +double_int_min_subnormal DOUBLE 0000000000000001 INT INT +double_int_max_finite DOUBLE 7fefffffffffffff INT INT +double_int_byte_neg_128 DOUBLE c060000000000000 INT INT ffffff80 -128 +double_int_byte_pos_127 DOUBLE 405fc00000000000 INT INT 0000007f 127 +double_int_byte_neg_129 DOUBLE c060200000000000 INT INT ffffff7f -129 +double_int_byte_pos_128 DOUBLE 4060000000000000 INT INT 00000080 128 +double_int_short_neg_32768 DOUBLE c0e0000000000000 INT INT ffff8000 -32768 +double_int_short_pos_32767 DOUBLE 40dfffc000000000 INT INT 00007fff 32767 +double_int_short_neg_32769 DOUBLE c0e0002000000000 INT INT ffff7fff -32769 +double_int_short_pos_32768 DOUBLE 40e0000000000000 INT INT 00008000 32768 +double_int_int_neg_2147483648 DOUBLE c1e0000000000000 INT INT 80000000 NULL +double_int_int_pos_2147483647 DOUBLE 41dfffffffc00000 INT INT 7fffffff 2147483647 +double_int_int_neg_2147483649 DOUBLE c1e0000000200000 INT INT +double_int_int_pos_2147483648 DOUBLE 41e0000000000000 INT INT +double_int_long_neg_9223372036854776000 DOUBLE c3e0000000000000 INT INT +double_int_long_pos_9223372036854776000 DOUBLE 43e0000000000000 INT INT +double_long_pos_zero DOUBLE 0000000000000000 LONG LONG 0000000000000000 0 +double_long_neg_zero DOUBLE 8000000000000000 LONG LONG 0000000000000000 0 +double_long_pos_one DOUBLE 3ff0000000000000 LONG LONG 0000000000000001 1 +double_long_neg_one DOUBLE bff0000000000000 LONG LONG ffffffffffffffff -1 +double_long_fraction DOUBLE 3ff8000000000000 LONG LONG +double_long_pos_inf DOUBLE 7ff0000000000000 LONG LONG +double_long_neg_inf DOUBLE fff0000000000000 LONG LONG +double_long_nan_payload DOUBLE 7ff8000000000001 LONG LONG NULL +double_long_nan_negative DOUBLE fff8000000000042 LONG LONG NULL +double_long_min_subnormal DOUBLE 0000000000000001 LONG LONG +double_long_max_finite DOUBLE 7fefffffffffffff LONG LONG +double_long_byte_neg_128 DOUBLE c060000000000000 LONG LONG ffffffffffffff80 -128 +double_long_byte_pos_127 DOUBLE 405fc00000000000 LONG LONG 000000000000007f 127 +double_long_byte_neg_129 DOUBLE c060200000000000 LONG LONG ffffffffffffff7f -129 +double_long_byte_pos_128 DOUBLE 4060000000000000 LONG LONG 0000000000000080 128 +double_long_short_neg_32768 DOUBLE c0e0000000000000 LONG LONG ffffffffffff8000 -32768 +double_long_short_pos_32767 DOUBLE 40dfffc000000000 LONG LONG 0000000000007fff 32767 +double_long_short_neg_32769 DOUBLE c0e0002000000000 LONG LONG ffffffffffff7fff -32769 +double_long_short_pos_32768 DOUBLE 40e0000000000000 LONG LONG 0000000000008000 32768 +double_long_int_neg_2147483648 DOUBLE c1e0000000000000 LONG LONG ffffffff80000000 -2147483648 +double_long_int_pos_2147483647 DOUBLE 41dfffffffc00000 LONG LONG 000000007fffffff 2147483647 +double_long_int_neg_2147483649 DOUBLE c1e0000000200000 LONG LONG ffffffff7fffffff -2147483649 +double_long_int_pos_2147483648 DOUBLE 41e0000000000000 LONG LONG 0000000080000000 2147483648 +double_long_long_neg_9223372036854776000 DOUBLE c3e0000000000000 LONG LONG 8000000000000000 NULL +double_long_long_pos_9223372036854776000 DOUBLE 43e0000000000000 LONG LONG +double_float_pos_zero DOUBLE 0000000000000000 FLOAT FLOAT 00000000 BITS:00000000 +double_float_neg_zero DOUBLE 8000000000000000 FLOAT FLOAT 80000000 BITS:80000000 +double_float_pos_one DOUBLE 3ff0000000000000 FLOAT FLOAT 3f800000 BITS:3f800000 +double_float_neg_one DOUBLE bff0000000000000 FLOAT FLOAT bf800000 BITS:bf800000 +double_float_fraction DOUBLE 3ff8000000000000 FLOAT FLOAT 3fc00000 BITS:3fc00000 +double_float_pos_inf DOUBLE 7ff0000000000000 FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_neg_inf DOUBLE fff0000000000000 FLOAT FLOAT ff800000 BITS:ff800000 +double_float_nan_payload DOUBLE 7ff8000000000001 FLOAT FLOAT NULL +double_float_nan_negative DOUBLE fff8000000000042 FLOAT FLOAT NULL +double_float_min_subnormal DOUBLE 0000000000000001 FLOAT FLOAT 00000000 BITS:00000000 +double_float_max_finite DOUBLE 7fefffffffffffff FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_byte_neg_128 DOUBLE c060000000000000 FLOAT FLOAT c3000000 BITS:c3000000 +double_float_byte_pos_127 DOUBLE 405fc00000000000 FLOAT FLOAT 42fe0000 BITS:42fe0000 +double_float_byte_neg_129 DOUBLE c060200000000000 FLOAT FLOAT c3010000 BITS:c3010000 +double_float_byte_pos_128 DOUBLE 4060000000000000 FLOAT FLOAT 43000000 BITS:43000000 +double_float_short_neg_32768 DOUBLE c0e0000000000000 FLOAT FLOAT c7000000 BITS:c7000000 +double_float_short_pos_32767 DOUBLE 40dfffc000000000 FLOAT FLOAT 46fffe00 BITS:46fffe00 +double_float_short_neg_32769 DOUBLE c0e0002000000000 FLOAT FLOAT c7000100 BITS:c7000100 +double_float_short_pos_32768 DOUBLE 40e0000000000000 FLOAT FLOAT 47000000 BITS:47000000 +double_float_int_neg_2147483648 DOUBLE c1e0000000000000 FLOAT FLOAT cf000000 BITS:cf000000 +double_float_int_pos_2147483647 DOUBLE 41dfffffffc00000 FLOAT FLOAT 4f000000 BITS:4f000000 +double_float_int_neg_2147483649 DOUBLE c1e0000000200000 FLOAT FLOAT cf000000 BITS:cf000000 +double_float_int_pos_2147483648 DOUBLE 41e0000000000000 FLOAT FLOAT 4f000000 BITS:4f000000 +double_float_long_neg_9223372036854776000 DOUBLE c3e0000000000000 FLOAT FLOAT df000000 BITS:df000000 +double_float_long_pos_9223372036854776000 DOUBLE 43e0000000000000 FLOAT FLOAT 5f000000 BITS:5f000000 +double_double_pos_zero DOUBLE 0000000000000000 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 +double_double_neg_zero DOUBLE 8000000000000000 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 +double_double_pos_one DOUBLE 3ff0000000000000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 +double_double_neg_one DOUBLE bff0000000000000 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 +double_double_fraction DOUBLE 3ff8000000000000 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 +double_double_pos_inf DOUBLE 7ff0000000000000 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 +double_double_neg_inf DOUBLE fff0000000000000 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 +double_double_nan_payload DOUBLE 7ff8000000000001 DOUBLE DOUBLE NULL +double_double_nan_negative DOUBLE fff8000000000042 DOUBLE DOUBLE NULL +double_double_min_subnormal DOUBLE 0000000000000001 DOUBLE DOUBLE 0000000000000001 BITS:0000000000000001 +double_double_max_finite DOUBLE 7fefffffffffffff DOUBLE DOUBLE 7fefffffffffffff BITS:7fefffffffffffff +double_double_byte_neg_128 DOUBLE c060000000000000 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 +double_double_byte_pos_127 DOUBLE 405fc00000000000 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 +double_double_byte_neg_129 DOUBLE c060200000000000 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 +double_double_byte_pos_128 DOUBLE 4060000000000000 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 +double_double_short_neg_32768 DOUBLE c0e0000000000000 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 +double_double_short_pos_32767 DOUBLE 40dfffc000000000 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 +double_double_short_neg_32769 DOUBLE c0e0002000000000 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 +double_double_short_pos_32768 DOUBLE 40e0000000000000 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 +double_double_int_neg_2147483648 DOUBLE c1e0000000000000 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 +double_double_int_pos_2147483647 DOUBLE 41dfffffffc00000 DOUBLE DOUBLE 41dfffffffc00000 BITS:41dfffffffc00000 +double_double_int_neg_2147483649 DOUBLE c1e0000000200000 DOUBLE DOUBLE c1e0000000200000 BITS:c1e0000000200000 +double_double_int_pos_2147483648 DOUBLE 41e0000000000000 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 +double_double_long_neg_9223372036854776000 DOUBLE c3e0000000000000 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 +double_double_long_pos_9223372036854776000 DOUBLE 43e0000000000000 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 +float_long_pos_below FLOAT 5effffff LONG LONG 7fffff8000000000 9223371487098961920 +float_long_pos_above FLOAT 5f000001 LONG LONG +float_long_neg_inside FLOAT deffffff LONG LONG 8000008000000000 -9223371487098961920 +float_long_neg_outside FLOAT df000001 LONG LONG +double_long_pos_below DOUBLE 43dfffffffffffff LONG LONG 7ffffffffffffc00 9223372036854774784 +double_long_pos_above DOUBLE 43e0000000000001 LONG LONG +double_long_neg_inside DOUBLE c3dfffffffffffff LONG LONG 8000000000000400 -9223372036854774784 +double_long_neg_outside DOUBLE c3e0000000000001 LONG LONG +float_int_pos_below FLOAT 4effffff INT INT 7fffff80 2147483520 +float_int_pos_at_2p31 FLOAT 4f000000 INT INT +float_int_neg_at_min FLOAT cf000000 INT INT 80000000 NULL +float_int_neg_outside FLOAT cf000001 INT INT +double_int_pos_at_max DOUBLE 41dfffffffc00000 INT INT 7fffffff 2147483647 +double_int_pos_outside DOUBLE 41e0000000000000 INT INT +double_int_neg_at_min DOUBLE c1e0000000000000 INT INT 80000000 NULL +double_int_neg_outside DOUBLE c1e0000000200000 INT INT +float_byte_negative_fraction FLOAT bf000000 BYTE BYTE +float_short_negative_fraction FLOAT bf000000 SHORT SHORT +float_int_negative_fraction FLOAT bf000000 INT INT +float_long_negative_fraction FLOAT bf000000 LONG LONG +float_float_negative_fraction FLOAT bf000000 FLOAT FLOAT bf000000 BITS:bf000000 +float_double_negative_fraction FLOAT bf000000 DOUBLE DOUBLE bfe0000000000000 BITS:bfe0000000000000 +double_byte_negative_fraction DOUBLE bfe0000000000000 BYTE BYTE +double_short_negative_fraction DOUBLE bfe0000000000000 SHORT SHORT +double_int_negative_fraction DOUBLE bfe0000000000000 INT INT +double_long_negative_fraction DOUBLE bfe0000000000000 LONG LONG +double_float_negative_fraction DOUBLE bfe0000000000000 FLOAT FLOAT bf000000 BITS:bf000000 +double_double_negative_fraction DOUBLE bfe0000000000000 DOUBLE DOUBLE bfe0000000000000 BITS:bfe0000000000000 +float_byte_signaling_nan FLOAT 7f800001 BYTE BYTE 0 +float_short_signaling_nan FLOAT 7f800001 SHORT SHORT 0 +float_int_signaling_nan FLOAT 7f800001 INT INT NULL +float_long_signaling_nan FLOAT 7f800001 LONG LONG NULL +float_float_signaling_nan FLOAT 7f800001 FLOAT FLOAT NULL +float_double_signaling_nan FLOAT 7f800001 DOUBLE DOUBLE NULL +double_byte_signaling_nan DOUBLE 7ff0000000000001 BYTE BYTE 0 +double_short_signaling_nan DOUBLE 7ff0000000000001 SHORT SHORT 0 +double_int_signaling_nan DOUBLE 7ff0000000000001 INT INT NULL +double_long_signaling_nan DOUBLE 7ff0000000000001 LONG LONG NULL +double_float_signaling_nan DOUBLE 7ff0000000000001 FLOAT FLOAT NULL +double_double_signaling_nan DOUBLE 7ff0000000000001 DOUBLE DOUBLE NULL +double_float_half_min_subnormal DOUBLE 3690000000000000 FLOAT FLOAT 00000000 BITS:00000000 +double_float_above_half_min_subnormal DOUBLE 3690000000000001 FLOAT FLOAT 00000001 BITS:00000001 +double_float_negative_half_min_subnormal DOUBLE b690000000000000 FLOAT FLOAT 80000000 BITS:80000000 +double_float_float_min_normal DOUBLE 3810000000000000 FLOAT FLOAT 00800000 BITS:00800000 +double_float_below_float_min_normal DOUBLE 380fffffffffffff FLOAT FLOAT 00800000 BITS:00800000 +double_float_float_overflow_tie DOUBLE 47effffff0000000 FLOAT FLOAT 7f800000 BITS:7f800000 +double_float_below_float_overflow DOUBLE 47efffffefffffff FLOAT FLOAT 7f7fffff BITS:7f7fffff +double_float_tie_one_even_down DOUBLE 3ff0000010000000 FLOAT FLOAT 3f800000 BITS:3f800000 +double_float_tie_one_even_up DOUBLE 3ff0000030000000 FLOAT FLOAT 3f800002 BITS:3f800002 +double_float_subnormal_normal_tie DOUBLE 380fffffe0000000 FLOAT FLOAT 00800000 BITS:00800000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv new file mode 100644 index 000000000..cdb47cb18 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/floating-to-text.tsv @@ -0,0 +1,49 @@ +# case_id input_type input_raw_bits target_type expected_wire_type expected_utf8_hex expected_sql_value +float_string_pos_zero FLOAT 00000000 STRING VARCHAR 302e30 0.0 +float_string_neg_zero FLOAT 80000000 STRING VARCHAR 2d302e30 -0.0 +float_string_one FLOAT 3f800000 STRING VARCHAR 312e30 1.0 +float_string_point_one FLOAT 3dcccccd STRING VARCHAR 302e3130303030303030313439303131363132 0.10000000149011612 +float_string_min_subnormal FLOAT 00000001 STRING VARCHAR 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_string_max_finite FLOAT 7f7fffff STRING VARCHAR 332e34303238323334363633383532383836453338 3.4028234663852886E38 +float_string_pos_infinity FLOAT 7f800000 STRING VARCHAR 496e66696e697479 Infinity +float_string_neg_infinity FLOAT ff800000 STRING VARCHAR 2d496e66696e697479 -Infinity +float_string_qnan FLOAT 7fc00002 STRING VARCHAR +float_string_snan_negative FLOAT ff800001 STRING VARCHAR +float_string_1e23 FLOAT 65a96816 STRING VARCHAR 392e393939393939373738313936333038453232 9.999999778196308E22 +float_varchar_pos_zero FLOAT 00000000 VARCHAR VARCHAR 302e30 0.0 +float_varchar_neg_zero FLOAT 80000000 VARCHAR VARCHAR 2d302e30 -0.0 +float_varchar_point_one FLOAT 3dcccccd VARCHAR VARCHAR 302e3130303030303030313439303131363132 0.10000000149011612 +float_varchar_min_subnormal FLOAT 00000001 VARCHAR VARCHAR 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_varchar_pos_infinity FLOAT 7f800000 VARCHAR VARCHAR 496e66696e697479 Infinity +float_varchar_qnan FLOAT 7fc00002 VARCHAR VARCHAR +float_symbol_pos_zero FLOAT 00000000 SYMBOL SYMBOL 302e30 0.0 +float_symbol_neg_zero FLOAT 80000000 SYMBOL SYMBOL 2d302e30 -0.0 +float_symbol_point_one FLOAT 3dcccccd SYMBOL SYMBOL 302e3130303030303030313439303131363132 0.10000000149011612 +float_symbol_min_subnormal FLOAT 00000001 SYMBOL SYMBOL 312e343031323938343634333234383137452d3435 1.401298464324817E-45 +float_symbol_neg_infinity FLOAT ff800000 SYMBOL SYMBOL 2d496e66696e697479 -Infinity +float_symbol_snan_negative FLOAT ff800001 SYMBOL SYMBOL +double_string_pos_zero DOUBLE 0000000000000000 STRING VARCHAR 302e30 0.0 +double_string_neg_zero DOUBLE 8000000000000000 STRING VARCHAR 2d302e30 -0.0 +double_string_one DOUBLE 3ff0000000000000 STRING VARCHAR 312e30 1.0 +double_string_neg_two_half DOUBLE c004000000000000 STRING VARCHAR 2d322e35 -2.5 +double_string_1e23 DOUBLE 44b52d02c7e14af6 STRING VARCHAR 312e30453233 1.0E23 +double_string_min_subnormal DOUBLE 0000000000000001 STRING VARCHAR 352e30452d333234 5.0E-324 +double_string_max_finite DOUBLE 7fefffffffffffff STRING VARCHAR 312e3739373639333133343836323331353745333038 1.7976931348623157E308 +double_string_pos_infinity DOUBLE 7ff0000000000000 STRING VARCHAR 496e66696e697479 Infinity +double_string_neg_infinity DOUBLE fff0000000000000 STRING VARCHAR 2d496e66696e697479 -Infinity +double_string_qnan DOUBLE 7ff8000000000002 STRING VARCHAR +double_string_snan_negative DOUBLE fff0000000000001 STRING VARCHAR +double_varchar_pos_zero DOUBLE 0000000000000000 VARCHAR VARCHAR 302e30 0.0 +double_varchar_neg_zero DOUBLE 8000000000000000 VARCHAR VARCHAR 2d302e30 -0.0 +double_varchar_neg_two_half DOUBLE c004000000000000 VARCHAR VARCHAR 2d322e35 -2.5 +double_varchar_1e23 DOUBLE 44b52d02c7e14af6 VARCHAR VARCHAR 312e30453233 1.0E23 +double_varchar_min_subnormal DOUBLE 0000000000000001 VARCHAR VARCHAR 352e30452d333234 5.0E-324 +double_varchar_pos_infinity DOUBLE 7ff0000000000000 VARCHAR VARCHAR 496e66696e697479 Infinity +double_varchar_qnan DOUBLE 7ff8000000000002 VARCHAR VARCHAR +double_symbol_pos_zero DOUBLE 0000000000000000 SYMBOL SYMBOL 302e30 0.0 +double_symbol_neg_zero DOUBLE 8000000000000000 SYMBOL SYMBOL 2d302e30 -0.0 +double_symbol_neg_two_half DOUBLE c004000000000000 SYMBOL SYMBOL 2d322e35 -2.5 +double_symbol_1e23 DOUBLE 44b52d02c7e14af6 SYMBOL SYMBOL 312e30453233 1.0E23 +double_symbol_max_finite DOUBLE 7fefffffffffffff SYMBOL SYMBOL 312e3739373639333133343836323331353745333038 1.7976931348623157E308 +double_symbol_neg_infinity DOUBLE fff0000000000000 SYMBOL SYMBOL 2d496e66696e697479 -Infinity +double_symbol_snan_negative DOUBLE fff0000000000001 SYMBOL SYMBOL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv new file mode 100644 index 000000000..e203958cc --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/integer-temporal-conversions.tsv @@ -0,0 +1,25 @@ +# case_id input_type input_value target_type expected_value +byte_date_min BYTE -128 DATE -128 +byte_date_max BYTE 127 DATE 127 +byte_timestamp_min BYTE -128 TIMESTAMP -128 +byte_timestamp_max BYTE 127 TIMESTAMP 127 +byte_timestamp_ns_min BYTE -128 TIMESTAMP_NS -128 +byte_timestamp_ns_max BYTE 127 TIMESTAMP_NS 127 +short_date_min SHORT -32768 DATE -32768 +short_date_max SHORT 32767 DATE 32767 +short_timestamp_min SHORT -32768 TIMESTAMP -32768 +short_timestamp_max SHORT 32767 TIMESTAMP 32767 +short_timestamp_ns_min SHORT -32768 TIMESTAMP_NS -32768 +short_timestamp_ns_max SHORT 32767 TIMESTAMP_NS 32767 +int_date_null INT -2147483648 DATE +int_date_negative INT -1 DATE -1 +int_date_max INT 2147483647 DATE 2147483647 +int_timestamp_null INT -2147483648 TIMESTAMP +int_timestamp_negative INT -1 TIMESTAMP -1 +int_timestamp_max INT 2147483647 TIMESTAMP 2147483647 +int_timestamp_ns_null INT -2147483648 TIMESTAMP_NS +int_timestamp_ns_negative INT -1 TIMESTAMP_NS -1 +int_timestamp_ns_max INT 2147483647 TIMESTAMP_NS 2147483647 +long_date_null LONG -9223372036854775808 DATE +long_date_min_value LONG -9223372036854775807 DATE -9223372036854775807 +long_date_max LONG 9223372036854775807 DATE 9223372036854775807 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv new file mode 100644 index 000000000..8044b25f9 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/ipv4-conversions.tsv @@ -0,0 +1,17 @@ +# case_id input_kind input outcome expected_hex expected_text +typed_null INT 0 NULL 00000000 +typed_low INT 1 VALUE 00000001 0.0.0.1 +typed_int_min INT -2147483648 VALUE 80000000 128.0.0.0 +typed_high_bit INT -1062731519 VALUE c0a80101 192.168.1.1 +typed_all_bits INT -1 VALUE ffffffff 255.255.255.255 +text_private TEXT 10.20.30.40 VALUE 0a141e28 10.20.30.40 +text_normalized TEXT .....1.2.3.4...... VALUE 01020304 1.2.3.4 +text_normalized_null TEXT .0.0.0.0. NULL 00000000 +text_null_literal TEXT null INVALID - - +text_null_literal_upper TEXT NULL INVALID - - +text_zero_literal TEXT 0.0.0.0 INVALID - - +text_empty TEXT INVALID - - +text_bad_octet TEXT 1.2.3.256 INVALID - - +text_bad_character TEXT 1.2.3.x INVALID - - +text_single_dot TEXT . INVALID - - +text_only_dots TEXT .... INVALID - - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv new file mode 100644 index 000000000..ecf131bd5 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-decimal.tsv @@ -0,0 +1,40 @@ +# case_id input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +8_positive_max 9 DECIMAL8 2 1 VALUE 000000000000005a - - - 9.0 +8_positive_over 10 DECIMAL8 2 1 INVALID - - - - - +8_negative_max -9 DECIMAL8 2 1 VALUE ffffffffffffffa6 - - - -9.0 +8_negative_over -10 DECIMAL8 2 1 INVALID - - - - - +16_positive_max 99 DECIMAL16 4 2 VALUE 00000000000026ac - - - 99.00 +16_positive_over 100 DECIMAL16 4 2 INVALID - - - - - +16_negative_max -99 DECIMAL16 4 2 VALUE ffffffffffffd954 - - - -99.00 +16_negative_over -100 DECIMAL16 4 2 INVALID - - - - - +32_positive_max 999999 DECIMAL32 9 3 VALUE 000000003b9ac618 - - - 999999.000 +32_positive_over 1000000 DECIMAL32 9 3 INVALID - - - - - +32_negative_max -999999 DECIMAL32 9 3 VALUE ffffffffc46539e8 - - - -999999.000 +32_negative_over -1000000 DECIMAL32 9 3 INVALID - - - - - +64_positive_max 99999999999999999 DECIMAL64 18 1 VALUE 0de0b6b3a763fff6 - - - 99999999999999999.0 +64_positive_over 100000000000000000 DECIMAL64 18 1 INVALID - - - - - +64_negative_max -99999999999999999 DECIMAL64 18 1 VALUE f21f494c589c000a - - - -99999999999999999.0 +64_negative_over -100000000000000000 DECIMAL64 18 1 INVALID - - - - - +128_positive_max 999999999999999999 DECIMAL128 38 20 VALUE 9dc2c4129cf00000 4b3b4ca85a86c474 - - 999999999999999999.00000000000000000000 +128_positive_over 1000000000000000000 DECIMAL128 38 20 INVALID - - - - - +128_negative_max -999999999999999999 DECIMAL128 38 20 VALUE 623d3bed63100000 b4c4b357a5793b8b - - -999999999999999999.00000000000000000000 +128_negative_over -1000000000000000000 DECIMAL128 38 20 INVALID - - - - - +256_positive_max 999999999999999999 DECIMAL256 75 57 VALUE b600000000000000 9ff45edd38440d7c 8b27fbc1cea32b80 0235fadd81c2822b 999999999999999999.000000000000000000000000000000000000000000000000000000000 +256_positive_over 1000000000000000000 DECIMAL256 75 57 INVALID - - - - - +256_negative_max -999999999999999999 DECIMAL256 75 57 VALUE 4a00000000000000 600ba122c7bbf283 74d8043e315cd47f fdca05227e3d7dd4 -999999999999999999.000000000000000000000000000000000000000000000000000000000 +256_negative_over -1000000000000000000 DECIMAL256 75 57 INVALID - - - - - +long_max_to_128 9223372036854775807 DECIMAL128 38 19 VALUE 7538dcfb76180000 4563918244f3ffff - - 9223372036854775807.0000000000000000000 +long_min_plus_one_to_128 -9223372036854775807 DECIMAL128 38 19 VALUE 8ac7230489e80000 ba9c6e7dbb0c0000 - - -9223372036854775807.0000000000000000000 +long_max_to_256 9223372036854775807 DECIMAL256 76 57 VALUE b600000000000000 b90234ab79b5257c cd3668f47a8b47cf 14643e5ae44d12b8 9223372036854775807.000000000000000000000000000000000000000000000000000000000 +long_min_plus_one_to_256 -9223372036854775807 DECIMAL256 76 57 VALUE 4a00000000000000 46fdcb54864ada83 32c9970b8574b830 eb9bc1a51bb2ed47 -9223372036854775807.000000000000000000000000000000000000000000000000000000000 +long_max_256_storage_over 9223372036854775807 DECIMAL256 76 58 INVALID - - - - - +long_min_plus_one_256_storage_over -9223372036854775807 DECIMAL256 76 58 INVALID - - - - - +zero_64_max_scale 0 DECIMAL64 18 18 VALUE 0000000000000000 - - - 0.000000000000000000 +zero_128_max_scale 0 DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +zero_256_max_scale 0 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +long_min_null_8 -9223372036854775808 DECIMAL8 2 1 NULL - - - - NULL +long_min_null_16 -9223372036854775808 DECIMAL16 4 2 NULL - - - - NULL +long_min_null_32 -9223372036854775808 DECIMAL32 9 3 NULL - - - - NULL +long_min_null_64 -9223372036854775808 DECIMAL64 18 1 NULL - - - - NULL +long_min_null_128 -9223372036854775808 DECIMAL128 38 20 NULL - - - - NULL +long_min_null_256 -9223372036854775808 DECIMAL256 76 58 NULL - - - - NULL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv new file mode 100644 index 000000000..53af275f1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-numeric.tsv @@ -0,0 +1,49 @@ +# case_id input target expected +# means the target's missing-value representation; BYTE/SHORT materialize it as zero. +long_min_byte -9223372036854775808 BYTE +long_min_short -9223372036854775808 SHORT +long_min_int -9223372036854775808 INT +long_min_long -9223372036854775808 LONG +long_min_float -9223372036854775808 FLOAT +long_min_double -9223372036854775808 DOUBLE +byte_min -128 BYTE -128 +byte_max 127 BYTE 127 +byte_zero 0 BYTE 0 +byte_below -129 BYTE +byte_above 128 BYTE +short_min -32768 SHORT -32768 +short_max 32767 SHORT 32767 +short_zero 0 SHORT 0 +short_below -32769 SHORT +short_above 32768 SHORT +int_min -2147483648 INT +int_min_plus_one -2147483647 INT -2147483647 +int_max 2147483647 INT 2147483647 +int_zero 0 INT 0 +int_below -2147483649 INT +int_above 2147483648 INT +zero_long 0 LONG 0 +minus_one_long -1 LONG -1 +long_max 9223372036854775807 LONG 9223372036854775807 +long_min_plus_one -9223372036854775807 LONG -9223372036854775807 +zero_float 0 FLOAT 0x00000000 +minus_one_float -1 FLOAT 0xbf800000 +precision_float 2147483647 FLOAT 0x4f000000 +float_exact_below 16777215 FLOAT 0x4b7fffff +float_exact_boundary 16777216 FLOAT 0x4b800000 +float_round_down 16777217 FLOAT 0x4b800000 +float_round_even 16777218 FLOAT 0x4b800001 +float_round_up 16777219 FLOAT 0x4b800002 +float_negative_round -16777217 FLOAT 0xcb800000 +long_min_plus_one_float -9223372036854775807 FLOAT 0xdf000000 +long_max_float 9223372036854775807 FLOAT 0x5f000000 +zero_double 0 DOUBLE 0x0000000000000000 +minus_one_double -1 DOUBLE 0xbff0000000000000 +precision_double 9007199254740993 DOUBLE 0x4340000000000000 +double_exact_below 9007199254740991 DOUBLE 0x433fffffffffffff +double_exact_boundary 9007199254740992 DOUBLE 0x4340000000000000 +double_next_even 9007199254740994 DOUBLE 0x4340000000000001 +double_round_up 9007199254740995 DOUBLE 0x4340000000000002 +double_negative_round -9007199254740993 DOUBLE 0xc340000000000000 +long_min_plus_one_double -9223372036854775807 DOUBLE 0xc3e0000000000000 +long_max_double 9223372036854775807 DOUBLE 0x43e0000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv new file mode 100644 index 000000000..23967eb15 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-text.tsv @@ -0,0 +1,19 @@ +# case_id input_long target_type expected_wire_type expected_utf8_hex expected_sql_value +long_string_min -9223372036854775808 STRING VARCHAR +long_string_min_plus_1 -9223372036854775807 STRING VARCHAR 2d39323233333732303336383534373735383037 -9223372036854775807 +long_string_neg_1 -1 STRING VARCHAR 2d31 -1 +long_string_zero 0 STRING VARCHAR 30 0 +long_string_one 1 STRING VARCHAR 31 1 +long_string_max 9223372036854775807 STRING VARCHAR 39323233333732303336383534373735383037 9223372036854775807 +long_varchar_min -9223372036854775808 VARCHAR VARCHAR +long_varchar_min_plus_1 -9223372036854775807 VARCHAR VARCHAR 2d39323233333732303336383534373735383037 -9223372036854775807 +long_varchar_neg_1 -1 VARCHAR VARCHAR 2d31 -1 +long_varchar_zero 0 VARCHAR VARCHAR 30 0 +long_varchar_one 1 VARCHAR VARCHAR 31 1 +long_varchar_max 9223372036854775807 VARCHAR VARCHAR 39323233333732303336383534373735383037 9223372036854775807 +long_symbol_min -9223372036854775808 SYMBOL SYMBOL +long_symbol_min_plus_1 -9223372036854775807 SYMBOL SYMBOL 2d39323233333732303336383534373735383037 -9223372036854775807 +long_symbol_neg_1 -1 SYMBOL SYMBOL 2d31 -1 +long_symbol_zero 0 SYMBOL SYMBOL 30 0 +long_symbol_one 1 SYMBOL SYMBOL 31 1 +long_symbol_max 9223372036854775807 SYMBOL SYMBOL 39323233333732303336383534373735383037 9223372036854775807 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv new file mode 100644 index 000000000..cdd0db95a --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long-to-timestamp.tsv @@ -0,0 +1,13 @@ +# case_id input_value target_type expected_wire_type expected_wire_value expected_sql_null +long_timestamp_min -9223372036854775808 TIMESTAMP TIMESTAMP true +long_timestamp_min_plus_one -9223372036854775807 TIMESTAMP TIMESTAMP -9223372036854775807 false +long_timestamp_negative_one -1 TIMESTAMP TIMESTAMP -1 false +long_timestamp_zero 0 TIMESTAMP TIMESTAMP 0 false +long_timestamp_one 1 TIMESTAMP TIMESTAMP 1 false +long_timestamp_max 9223372036854775807 TIMESTAMP TIMESTAMP 9223372036854775807 false +long_timestamp_nanos_min -9223372036854775808 TIMESTAMP_NANOS TIMESTAMP_NANOS true +long_timestamp_nanos_min_plus_one -9223372036854775807 TIMESTAMP_NANOS TIMESTAMP_NANOS -9223372036854775807 false +long_timestamp_nanos_negative_one -1 TIMESTAMP_NANOS TIMESTAMP_NANOS -1 false +long_timestamp_nanos_zero 0 TIMESTAMP_NANOS TIMESTAMP_NANOS 0 false +long_timestamp_nanos_one 1 TIMESTAMP_NANOS TIMESTAMP_NANOS 1 false +long_timestamp_nanos_max 9223372036854775807 TIMESTAMP_NANOS TIMESTAMP_NANOS 9223372036854775807 false diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv new file mode 100644 index 000000000..9b45988c2 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/long256-conversions.tsv @@ -0,0 +1,9 @@ +# case_id input_l0 input_l1 input_l2 input_l3 outcome expected_text +zero 0 0 0 0 VALUE 0x00 +low_one 1 0 0 0 VALUE 0x01 +second_limb 1 2 0 0 VALUE 0x020000000000000001 +limb_order 1 2 3 4 VALUE 0x04000000000000000300000000000000020000000000000001 +all_ones -1 -1 -1 -1 VALUE 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff +partial_min -9223372036854775808 -9223372036854775808 -9223372036854775808 0 VALUE 0x800000000000000080000000000000008000000000000000 +high_sign_bit 0 0 0 -9223372036854775808 VALUE 0x8000000000000000000000000000000000000000000000000000000000000000 +all_min_null -9223372036854775808 -9223372036854775808 -9223372036854775808 -9223372036854775808 NULL - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv new file mode 100644 index 000000000..caa4173ee --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-conversions.tsv @@ -0,0 +1,61 @@ +# case_id source_type source_scale source_ll_hex source_lh_hex source_hl_hex source_hh_hex target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +64_to_8 DECIMAL64 1 000000000000000c - - - DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +64_to_16 DECIMAL64 1 000000000000000c - - - DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +64_to_32 DECIMAL64 1 000000000000000c - - - DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +64_to_64 DECIMAL64 1 000000000000000c - - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +64_to_128 DECIMAL64 1 000000000000000c - - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +64_to_256 DECIMAL64 1 000000000000000c - - - DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +128_to_8 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +128_to_16 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +128_to_32 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +128_to_64 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +128_to_128 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +128_to_256 DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +256_to_8 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL8 2 1 VALUE 000000000000000c - - - 1.2 +256_to_16 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL16 4 1 VALUE 000000000000000c - - - 1.2 +256_to_32 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL32 9 1 VALUE 000000000000000c - - - 1.2 +256_to_64 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +256_to_128 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +256_to_256 DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +64_zero DECIMAL64 2 0000000000000000 - - - DECIMAL64 18 2 VALUE 0000000000000000 - - - 0.00 +64_negative DECIMAL64 2 fffffffffffffff4 - - - DECIMAL64 18 2 VALUE fffffffffffffff4 - - - -0.12 +64_upscale DECIMAL64 1 000000000000000c - - - DECIMAL64 18 3 VALUE 00000000000004b0 - - - 1.200 +64_downscale_exact DECIMAL64 3 00000000000004b0 - - - DECIMAL64 18 1 VALUE 000000000000000c - - - 1.2 +64_downscale_inexact DECIMAL64 3 00000000000004b1 - - - DECIMAL64 18 1 INVALID - - - - - +128_zero DECIMAL128 2 0000000000000000 0000000000000000 - - DECIMAL128 38 2 VALUE 0000000000000000 0000000000000000 - - 0.00 +128_negative DECIMAL128 2 fffffffffffffff4 ffffffffffffffff - - DECIMAL128 38 2 VALUE fffffffffffffff4 ffffffffffffffff - - -0.12 +128_upscale DECIMAL128 1 000000000000000c 0000000000000000 - - DECIMAL128 38 3 VALUE 00000000000004b0 0000000000000000 - - 1.200 +128_downscale_exact DECIMAL128 3 00000000000004b0 0000000000000000 - - DECIMAL128 38 1 VALUE 000000000000000c 0000000000000000 - - 1.2 +128_downscale_inexact DECIMAL128 3 00000000000004b1 0000000000000000 - - DECIMAL128 38 1 INVALID - - - - - +256_zero DECIMAL256 2 0000000000000000 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 2 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.00 +256_negative DECIMAL256 2 fffffffffffffff4 ffffffffffffffff ffffffffffffffff ffffffffffffffff DECIMAL256 76 2 VALUE fffffffffffffff4 ffffffffffffffff ffffffffffffffff ffffffffffffffff -0.12 +256_upscale DECIMAL256 1 000000000000000c 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 3 VALUE 00000000000004b0 0000000000000000 0000000000000000 0000000000000000 1.200 +256_downscale_exact DECIMAL256 3 00000000000004b0 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 VALUE 000000000000000c 0000000000000000 0000000000000000 0000000000000000 1.2 +256_downscale_inexact DECIMAL256 3 00000000000004b1 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 1 INVALID - - - - - +8_precision_max DECIMAL64 0 0000000000000063 - - - DECIMAL8 2 0 VALUE 0000000000000063 - - - 99 +8_precision_overflow DECIMAL64 0 0000000000000064 - - - DECIMAL8 2 0 INVALID - - - - - +16_precision_max DECIMAL64 0 000000000000270f - - - DECIMAL16 4 0 VALUE 000000000000270f - - - 9999 +16_precision_overflow DECIMAL64 0 0000000000002710 - - - DECIMAL16 4 0 INVALID - - - - - +32_precision_max DECIMAL64 0 000000003b9ac9ff - - - DECIMAL32 9 0 VALUE 000000003b9ac9ff - - - 999999999 +32_precision_overflow DECIMAL64 0 000000003b9aca00 - - - DECIMAL32 9 0 INVALID - - - - - +64_precision_max DECIMAL64 0 0de0b6b3a763ffff - - - DECIMAL64 18 0 VALUE 0de0b6b3a763ffff - - - 999999999999999999 +64_precision_overflow DECIMAL64 0 0de0b6b3a7640000 - - - DECIMAL64 18 0 INVALID - - - - - +128_precision_max DECIMAL128 0 098a223fffffffff 4b3b4ca85a86c47a - - DECIMAL128 38 0 VALUE 098a223fffffffff 4b3b4ca85a86c47a - - 99999999999999999999999999999999999999 +128_precision_overflow DECIMAL128 0 098a224000000000 4b3b4ca85a86c47a - - DECIMAL128 38 0 INVALID - - - - - +256_precision_max DECIMAL256 0 ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 DECIMAL256 76 0 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_precision_overflow DECIMAL256 0 0000000000000000 7775a5f171951000 0764b4abe8652979 161bcca7119915b5 DECIMAL256 76 0 INVALID - - - - - +8_precision_negative_max DECIMAL64 0 ffffffffffffff9d - - - DECIMAL8 2 0 VALUE ffffffffffffff9d - - - -99 +8_precision_negative_overflow DECIMAL64 0 ffffffffffffff9c - - - DECIMAL8 2 0 INVALID - - - - - +16_precision_negative_max DECIMAL64 0 ffffffffffffd8f1 - - - DECIMAL16 4 0 VALUE ffffffffffffd8f1 - - - -9999 +16_precision_negative_overflow DECIMAL64 0 ffffffffffffd8f0 - - - DECIMAL16 4 0 INVALID - - - - - +32_precision_negative_max DECIMAL64 0 ffffffffc4653601 - - - DECIMAL32 9 0 VALUE ffffffffc4653601 - - - -999999999 +32_precision_negative_overflow DECIMAL64 0 ffffffffc4653600 - - - DECIMAL32 9 0 INVALID - - - - - +64_precision_negative_max DECIMAL64 0 f21f494c589c0001 - - - DECIMAL64 18 0 VALUE f21f494c589c0001 - - - -999999999999999999 +64_precision_negative_overflow DECIMAL64 0 f21f494c589c0000 - - - DECIMAL64 18 0 INVALID - - - - - +128_precision_negative_max DECIMAL128 0 f675ddc000000001 b4c4b357a5793b85 - - DECIMAL128 38 0 VALUE f675ddc000000001 b4c4b357a5793b85 - - -99999999999999999999999999999999999999 +128_precision_negative_overflow DECIMAL128 0 f675ddc000000000 b4c4b357a5793b85 - - DECIMAL128 38 0 INVALID - - - - - +256_precision_negative_max DECIMAL256 0 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a DECIMAL256 76 0 VALUE 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a -9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_precision_negative_overflow DECIMAL256 0 0000000000000000 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a DECIMAL256 76 0 INVALID - - - - - +64_zero_max_scale DECIMAL64 18 0000000000000000 - - - DECIMAL64 18 18 VALUE 0000000000000000 - - - 0.000000000000000000 +128_zero_max_scale DECIMAL128 38 0000000000000000 0000000000000000 - - DECIMAL128 38 38 VALUE 0000000000000000 0000000000000000 - - 0.00000000000000000000000000000000000000 +256_zero_max_scale DECIMAL256 76 0000000000000000 0000000000000000 0000000000000000 0000000000000000 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv new file mode 100644 index 000000000..40af5fb0f --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-decimal-to-text.tsv @@ -0,0 +1,25 @@ +# case_id source_type source_scale source_ll_hex source_lh_hex source_hl_hex source_hh_hex target_type input_form expected_text +64_string_positive_trailing DECIMAL64 2 0000000000003034 - - - STRING VALUE 123.40 +64_varchar_negative_fraction DECIMAL64 3 ffffffffffffff83 - - - VARCHAR VALUE -0.125 +64_string_precision_max DECIMAL64 0 0de0b6b3a763ffff - - - STRING VALUE 999999999999999999 +64_varchar_negative_max_scale DECIMAL64 18 f21f494c589c0001 - - - VARCHAR VALUE -0.999999999999999999 +64_string_leading_fraction DECIMAL64 18 0000000000000001 - - - STRING VALUE 0.000000000000000001 +64_varchar_zero_max_scale DECIMAL64 18 0000000000000000 - - - VARCHAR VALUE 0.000000000000000000 +128_string_positive_trailing DECIMAL128 2 0000000000003034 0000000000000000 - - STRING VALUE 123.40 +128_varchar_negative_fraction DECIMAL128 3 ffffffffffffff83 ffffffffffffffff - - VARCHAR VALUE -0.125 +128_string_precision_max DECIMAL128 0 098a223fffffffff 4b3b4ca85a86c47a - - STRING VALUE 99999999999999999999999999999999999999 +128_varchar_negative_max_scale DECIMAL128 38 f675ddc000000001 b4c4b357a5793b85 - - VARCHAR VALUE -0.99999999999999999999999999999999999999 +128_string_leading_fraction DECIMAL128 38 0000000000000001 0000000000000000 - - STRING VALUE 0.00000000000000000000000000000000000001 +128_varchar_zero_max_scale DECIMAL128 38 0000000000000000 0000000000000000 - - VARCHAR VALUE 0.00000000000000000000000000000000000000 +256_string_positive_trailing DECIMAL256 2 0000000000003034 0000000000000000 0000000000000000 0000000000000000 STRING VALUE 123.40 +256_varchar_negative_fraction DECIMAL256 3 ffffffffffffff83 ffffffffffffffff ffffffffffffffff ffffffffffffffff VARCHAR VALUE -0.125 +256_string_precision_max DECIMAL256 0 ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 STRING VALUE 9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_varchar_negative_max_scale DECIMAL256 76 0000000000000001 888a5a0e8e6af000 f89b4b54179ad686 e9e43358ee66ea4a VARCHAR VALUE -0.9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_string_leading_fraction DECIMAL256 76 0000000000000001 0000000000000000 0000000000000000 0000000000000000 STRING VALUE 0.0000000000000000000000000000000000000000000000000000000000000000000000000001 +256_varchar_zero_max_scale DECIMAL256 76 0000000000000000 0000000000000000 0000000000000000 0000000000000000 VARCHAR VALUE 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 +64_string_java_null DECIMAL64 0 - - - - STRING JAVA_NULL - +64_varchar_null_value DECIMAL64 0 - - - - VARCHAR NULL_VALUE - +128_string_java_null DECIMAL128 0 - - - - STRING JAVA_NULL - +128_varchar_null_value DECIMAL128 0 - - - - VARCHAR NULL_VALUE - +256_string_java_null DECIMAL256 0 - - - - STRING JAVA_NULL - +256_varchar_null_value DECIMAL256 0 - - - - VARCHAR NULL_VALUE - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv new file mode 100644 index 000000000..484515cc0 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/native-geohash-conversions.tsv @@ -0,0 +1,16 @@ +# case_id input_bits precision_bits expected_value_hex expected_text +zero_1 0 1 0 0 +one_1 1 1 1 1 +zero_5 0 5 0 00000 +masked_5 -1 5 1f 11111 +max_7 127 7 7f 1111111 +max_8 255 8 ff 11111111 +max_15 32767 15 7fff 111111111111111 +max_16 65535 16 ffff 1111111111111111 +leading_zero_20 74565 20 12345 00010010001101000101 +pattern_20 703710 20 abcde 10101011110011011110 +max_31 2147483647 31 7fffffff 1111111111111111111111111111111 +max_32 4294967295 32 ffffffff 11111111111111111111111111111111 +max_35 34359738367 35 7ffffffff 11111111111111111111111111111111111 +masked_20 1152921504606846975 20 fffff 11111111111111111111 +max_60 1152921504606846975 60 fffffffffffffff 111111111111111111111111111111111111111111111111111111111111 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv new file mode 100644 index 000000000..89f5b126d --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-decimal.tsv @@ -0,0 +1,31 @@ +# case_id input_type input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +dec8_byte_pos BYTE 9 DECIMAL8 2 1 VALUE 000000000000005a - - - 9.0 +dec8_short_neg SHORT -9 DECIMAL8 2 1 VALUE ffffffffffffffa6 - - - -9.0 +dec8_int_pos INT 5 DECIMAL8 2 1 VALUE 0000000000000032 - - - 5.0 +dec8_int_null INT -2147483648 DECIMAL8 2 1 NULL - - - - NULL +dec16_byte_pos BYTE 99 DECIMAL16 4 2 VALUE 00000000000026ac - - - 99.00 +dec16_short_neg SHORT -99 DECIMAL16 4 2 VALUE ffffffffffffd954 - - - -99.00 +dec16_int_pos INT 42 DECIMAL16 4 2 VALUE 0000000000001068 - - - 42.00 +dec16_int_null INT -2147483648 DECIMAL16 4 2 NULL - - - - NULL +dec32_byte_min BYTE -128 DECIMAL32 9 3 VALUE fffffffffffe0c00 - - - -128.000 +dec32_short_max SHORT 32767 DECIMAL32 9 3 VALUE 0000000001f3fc18 - - - 32767.000 +dec32_int_neg INT -999999 DECIMAL32 9 3 VALUE ffffffffc46539e8 - - - -999999.000 +dec32_int_null INT -2147483648 DECIMAL32 9 3 NULL - - - - NULL +dec64_byte_max BYTE 127 DECIMAL64 18 2 VALUE 000000000000319c - - - 127.00 +dec64_short_min SHORT -32768 DECIMAL64 18 2 VALUE ffffffffffce0000 - - - -32768.00 +dec64_int_max INT 2147483647 DECIMAL64 18 2 VALUE 00000031ffffff9c - - - 2147483647.00 +dec64_int_null INT -2147483648 DECIMAL64 18 2 NULL - - - - NULL +dec128_byte_min BYTE -128 DECIMAL128 38 20 VALUE 1c50e94e78000000 fffffffffffffd4a - - -128.00000000000000000000 +dec128_short_max SHORT 32767 DECIMAL128 38 20 VALUE 434f535a9cf00000 000000000002b5de - - 32767.00000000000000000000 +dec128_int_min_plus_one INT -2147483647 DECIMAL128 38 20 VALUE ba3f5e2d63100000 fffffffd4a1c50ee - - -2147483647.00000000000000000000 +dec128_int_null INT -2147483648 DECIMAL128 38 20 NULL - - - - NULL +dec256_byte_max BYTE 127 DECIMAL256 76 57 VALUE b600000000000000 12e7deee9f226721 3b75de2e84789384 0000000000000014 127.000000000000000000000000000000000000000000000000000000000 +dec256_short_min SHORT -32768 DECIMAL256 76 57 VALUE 0000000000000000 1a55bcda92be5b00 c1a51bb2ed470a01 ffffffffffffeb9b -32768.000000000000000000000000000000000000000000000000000000000 +dec256_int_max INT 2147483647 DECIMAL256 76 57 VALUE b600000000000000 5727a1ed1eb5257c bb8496032d64c038 0000000014643e5a 2147483647.000000000000000000000000000000000000000000000000000000000 +dec256_int_null INT -2147483648 DECIMAL256 76 57 NULL - - - - NULL +byte_min_scale_zero BYTE -128 DECIMAL16 3 0 VALUE ffffffffffffff80 - - - -128 +short_min_scale_zero SHORT -32768 DECIMAL32 5 0 VALUE ffffffffffff8000 - - - -32768 +int_min_plus_one_scale_zero INT -2147483647 DECIMAL64 10 0 VALUE ffffffff80000001 - - - -2147483647 +byte_precision_overflow BYTE 10 DECIMAL8 2 1 INVALID - - - - - +short_precision_overflow SHORT 100 DECIMAL16 4 2 INVALID - - - - - +int_precision_overflow INT 1000000 DECIMAL32 9 3 INVALID - - - - - diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv new file mode 100644 index 000000000..e429960b1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-numeric.tsv @@ -0,0 +1,66 @@ +# case_id input_type input target expected +# FLOAT and DOUBLE expectations are exact raw IEEE-754 bits. INT_MIN is a stable source NULL. +byte_min_byte BYTE -128 BYTE -128 +byte_min_short BYTE -128 SHORT -128 +byte_min_int BYTE -128 INT -128 +byte_min_long BYTE -128 LONG -128 +byte_min_float BYTE -128 FLOAT 0xc3000000 +byte_min_double BYTE -128 DOUBLE 0xc060000000000000 +byte_zero_byte BYTE 0 BYTE 0 +byte_zero_short BYTE 0 SHORT 0 +byte_zero_int BYTE 0 INT 0 +byte_zero_long BYTE 0 LONG 0 +byte_zero_float BYTE 0 FLOAT 0x00000000 +byte_zero_double BYTE 0 DOUBLE 0x0000000000000000 +byte_max_byte BYTE 127 BYTE 127 +byte_max_short BYTE 127 SHORT 127 +byte_max_int BYTE 127 INT 127 +byte_max_long BYTE 127 LONG 127 +byte_max_float BYTE 127 FLOAT 0x42fe0000 +byte_max_double BYTE 127 DOUBLE 0x405fc00000000000 +short_min_byte SHORT -32768 BYTE +short_min_short SHORT -32768 SHORT -32768 +short_min_int SHORT -32768 INT -32768 +short_min_long SHORT -32768 LONG -32768 +short_min_float SHORT -32768 FLOAT 0xc7000000 +short_min_double SHORT -32768 DOUBLE 0xc0e0000000000000 +short_below_byte SHORT -129 BYTE +short_byte_min SHORT -128 BYTE -128 +short_byte_max SHORT 127 BYTE 127 +short_above_byte SHORT 128 BYTE +short_max_byte SHORT 32767 BYTE +short_max_short SHORT 32767 SHORT 32767 +short_max_int SHORT 32767 INT 32767 +short_max_long SHORT 32767 LONG 32767 +short_max_float SHORT 32767 FLOAT 0x46fffe00 +short_max_double SHORT 32767 DOUBLE 0x40dfffc000000000 +int_min_byte INT -2147483648 BYTE +int_min_short INT -2147483648 SHORT +int_min_int INT -2147483648 INT +int_min_long INT -2147483648 LONG +int_min_float INT -2147483648 FLOAT +int_min_double INT -2147483648 DOUBLE +int_min_plus_one_byte INT -2147483647 BYTE +int_min_plus_one_short INT -2147483647 SHORT +int_min_plus_one_int INT -2147483647 INT -2147483647 +int_min_plus_one_long INT -2147483647 LONG -2147483647 +int_min_plus_one_float INT -2147483647 FLOAT 0xcf000000 +int_min_plus_one_double INT -2147483647 DOUBLE 0xc1dfffffffc00000 +int_below_short INT -32769 SHORT +int_short_min INT -32768 SHORT -32768 +int_below_byte INT -129 BYTE +int_byte_min INT -128 BYTE -128 +int_byte_max INT 127 BYTE 127 +int_above_byte INT 128 BYTE +int_128_short INT 128 SHORT 128 +int_short_max INT 32767 SHORT 32767 +int_above_short INT 32768 SHORT +int_32768_int INT 32768 INT 32768 +int_float_round INT 16777217 FLOAT 0x4b800000 +int_double_exact INT 16777217 DOUBLE 0x4170000010000000 +int_max_byte INT 2147483647 BYTE +int_max_short INT 2147483647 SHORT +int_max_int INT 2147483647 INT 2147483647 +int_max_long INT 2147483647 LONG 2147483647 +int_max_float INT 2147483647 FLOAT 0x4f000000 +int_max_double INT 2147483647 DOUBLE 0x41dfffffffc00000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv new file mode 100644 index 000000000..32b77a1b5 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/small-integer-to-text.tsv @@ -0,0 +1,31 @@ +# case_id input_type input_value target_type expected_utf8_hex expected_sql_value +byte_string_min BYTE -128 STRING 2d313238 -128 +byte_string_max BYTE 127 STRING 313237 127 +short_string_min SHORT -32768 STRING 2d3332373638 -32768 +short_string_max SHORT 32767 STRING 3332373637 32767 +int_string_null INT -2147483648 STRING +int_string_min_value INT -2147483647 STRING 2d32313437343833363437 -2147483647 +int_string_neg_one INT -1 STRING 2d31 -1 +int_string_zero INT 0 STRING 30 0 +int_string_one INT 1 STRING 31 1 +int_string_max INT 2147483647 STRING 32313437343833363437 2147483647 +byte_varchar_min BYTE -128 VARCHAR 2d313238 -128 +byte_varchar_max BYTE 127 VARCHAR 313237 127 +short_varchar_min SHORT -32768 VARCHAR 2d3332373638 -32768 +short_varchar_max SHORT 32767 VARCHAR 3332373637 32767 +int_varchar_null INT -2147483648 VARCHAR +int_varchar_min_value INT -2147483647 VARCHAR 2d32313437343833363437 -2147483647 +int_varchar_neg_one INT -1 VARCHAR 2d31 -1 +int_varchar_zero INT 0 VARCHAR 30 0 +int_varchar_one INT 1 VARCHAR 31 1 +int_varchar_max INT 2147483647 VARCHAR 32313437343833363437 2147483647 +byte_symbol_min BYTE -128 SYMBOL 2d313238 -128 +byte_symbol_max BYTE 127 SYMBOL 313237 127 +short_symbol_min SHORT -32768 SYMBOL 2d3332373638 -32768 +short_symbol_max SHORT 32767 SYMBOL 3332373637 32767 +int_symbol_null INT -2147483648 SYMBOL +int_symbol_min_value INT -2147483647 SYMBOL 2d32313437343833363437 -2147483647 +int_symbol_neg_one INT -1 SYMBOL 2d31 -1 +int_symbol_zero INT 0 SYMBOL 30 0 +int_symbol_one INT 1 SYMBOL 31 1 +int_symbol_max INT 2147483647 SYMBOL 32313437343833363437 2147483647 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv new file mode 100644 index 000000000..0b31a9fdd --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-geohash-conversions.tsv @@ -0,0 +1,19 @@ +# case_id input bits outcome value_hex +bit1_max z 1 VALUE 1 +bit7_max zz 7 VALUE 7f +bit8_max zz 8 VALUE ff +bit15_max zzz 15 VALUE 7fff +bit16_max zzzz 16 VALUE ffff +bit31_max zzzzzzz 31 VALUE 7fffffff +bit32_max zzzzzzz 32 VALUE ffffffff +bit60_max zzzzzzzzzzzz 60 VALUE fffffffffffffff +uppercase U 5 VALUE 1a +zero 0 5 VALUE 0 +whole_prefix_consumed 10 5 VALUE 1 +suffix_after_cap zzzzzzzzzzzz! 60 VALUE fffffffffffffff +unicode_after_cap zzzzzzzzzzzzé 60 VALUE fffffffffffffff +source_null 8 NULL +empty 8 NULL +too_short zzzzzzzzzzz 60 INVALID +invalid_consumed a 5 INVALID +unicode_consumed é 5 INVALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv new file mode 100644 index 000000000..943b815d9 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-long256-conversions.tsv @@ -0,0 +1,24 @@ +# case_id input outcome l0_hex l1_hex l2_hex l3_hex +zero 0x00 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 +byte_max 0xff VALUE 00000000000000ff 0000000000000000 0000000000000000 0000000000000000 +uppercase_digits 0xABCD VALUE 000000000000abcd 0000000000000000 0000000000000000 0000000000000000 +leading_zeroes 0x0001 VALUE 0000000000000001 0000000000000000 0000000000000000 0000000000000000 +two_limbs 0x0123456789abcdeffedcba9876543210 VALUE fedcba9876543210 0123456789abcdef 0000000000000000 0000000000000000 +four_limbs 0x0123456789abcdeffedcba9876543210112233445566778899aabbccddeeff00 VALUE 99aabbccddeeff00 1122334455667788 fedcba9876543210 0123456789abcdef +all_ones 0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff VALUE ffffffffffffffff ffffffffffffffff ffffffffffffffff ffffffffffffffff +single_min_limb 0x8000000000000000 VALUE 8000000000000000 0000000000000000 0000000000000000 0000000000000000 +partial_high_group 0x00112233445566778899 VALUE 2233445566778899 0000000000000011 0000000000000000 0000000000000000 +three_limbs 0x111111111111111122222222222222223333333333333333 VALUE 3333333333333333 2222222222222222 1111111111111111 0000000000000000 +source_null NULL +empty_string INVALID +empty_digits 0x INVALID +odd_digits 0x0 INVALID +uppercase_prefix 0X00 INVALID +leading_sign -0x00 INVALID +leading_space 0x00 INVALID +internal_underscore 0x0_0 INVALID +non_hex 0xgg INVALID +non_ascii 0xéé INVALID +trailing_space 0x00 INVALID +too_long 0x000000000000000000000000000000000000000000000000000000000000000000 INVALID +null_sentinel 0x8000000000000000800000000000000080000000000000008000000000000000 INVALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv new file mode 100644 index 000000000..9be92bbd1 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-boolean.tsv @@ -0,0 +1,81 @@ +# case_id input_utf16_hex expected_wire_value expected_sql_value +true_case_00 0074007200750065 1 true +true_case_01 0054007200750065 1 true +true_case_02 0074005200750065 1 true +true_case_03 0054005200750065 1 true +true_case_04 0074007200550065 1 true +true_case_05 0054007200550065 1 true +true_case_06 0074005200550065 1 true +true_case_07 0054005200550065 1 true +true_case_08 0074007200750045 1 true +true_case_09 0054007200750045 1 true +true_case_10 0074005200750045 1 true +true_case_11 0054005200750045 1 true +true_case_12 0074007200550045 1 true +true_case_13 0054007200550045 1 true +true_case_14 0074005200550045 1 true +true_case_15 0054005200550045 1 true +false_case_00 00660061006c00730065 0 false +false_case_01 00460061006c00730065 0 false +false_case_02 00660041006c00730065 0 false +false_case_03 00460041006c00730065 0 false +false_case_04 00660061004c00730065 0 false +false_case_05 00460061004c00730065 0 false +false_case_06 00660041004c00730065 0 false +false_case_07 00460041004c00730065 0 false +false_case_08 00660061006c00530065 0 false +false_case_09 00460061006c00530065 0 false +false_case_10 00660041006c00530065 0 false +false_case_11 00460041006c00530065 0 false +false_case_12 00660061004c00530065 0 false +false_case_13 00460061004c00530065 0 false +false_case_14 00660041004c00530065 0 false +false_case_15 00460041004c00530065 0 false +false_case_16 00660061006c00730045 0 false +false_case_17 00460061006c00730045 0 false +false_case_18 00660041006c00730045 0 false +false_case_19 00460041006c00730045 0 false +false_case_20 00660061004c00730045 0 false +false_case_21 00460061004c00730045 0 false +false_case_22 00660041004c00730045 0 false +false_case_23 00460041004c00730045 0 false +false_case_24 00660061006c00530045 0 false +false_case_25 00460061006c00530045 0 false +false_case_26 00660041006c00530045 0 false +false_case_27 00460041006c00530045 0 false +false_case_28 00660061004c00530045 0 false +false_case_29 00460061004c00530045 0 false +false_case_30 00660041004c00530045 0 false +false_case_31 00460041004c00530045 0 false +digit_zero 0030 0 false +digit_one 0031 1 true +explicit_null false +empty +leading_space 00200074007200750065 +trailing_space 00740072007500650020 +leading_tab 00090074007200750065 +trailing_lf 0074007200750065000a +double_zero 00300030 +zero_one 00300031 +one_zero 00310030 +plus_one 002b0031 +minus_zero 002d0030 +yes 007900650073 +truth 00740072007500740068 +falsehood 00660061006c007300650068006f006f0064 +unicode_capital_t 0166007200750065 +fullwidth_one ff11 +nul_true 00000074007200750065 +lone_high d800 +lone_low dc00 +true_nul 00740072007500650000 +single_t 0074 +single_f 0066 +no 006e006f +null_word 006e0075006c006c +digit_two 0032 +minus_one 002d0031 +true_question 0074007200750065003f +spaces_only 00200020 +false_long_s 00660061006c017f0065 +true_embedded_high 00740072d8000065 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv new file mode 100644 index 000000000..d7c321ebb --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-date.tsv @@ -0,0 +1,54 @@ +case_id input_kind input outcome expected_raw feature +java_null TEXT NULL java_null +empty TEXT INVALID empty +raw_zero TEXT 0 VALID 0 raw_millis +raw_negative TEXT -1 VALID -1 raw_millis +raw_suffix TEXT 1577836800000L VALID 1577836800000 raw_suffix +raw_underscore TEXT 1_000 VALID 1000 raw_underscore +raw_null_sentinel TEXT -9223372036854775808 NULL raw_null_sentinel +raw_max TEXT 9223372036854775807 VALID 9223372036854775807 raw_boundary +raw_overflow TEXT 9223372036854775808 INVALID raw_overflow +epoch_date TEXT 1970-01-01 VALID 0 pg_date +pre_epoch_date TEXT 1969-12-31 VALID -86400000 pg_date +leap_date TEXT 2000-02-29 VALID 951782400000 pg_date +non_leap_date TEXT 1900-02-29 INVALID calendar_validation +date_time_utc TEXT 2020-01-01 00:00:00UTC VALID 1577836800000 pg_datetime +date_zone_utc TEXT 2020-01-01 UTC VALID 1577836800000 pg_date_zone +fraction_one TEXT 2020-01-01 00:00:00.1UTC VALID 1577836800100 greedy_millis +fraction_two TEXT 2020-01-01 00:00:00.12UTC VALID 1577836800120 greedy_millis +fraction_three TEXT 2020-01-01 00:00:00.123UTC VALID 1577836800123 greedy_millis +fraction_negative TEXT 2020-01-01 00:00:00.-1UTC VALID 1577836799990 greedy_millis +fraction_four TEXT 2020-01-01 00:00:00.1234UTC INVALID greedy_millis +iso_z TEXT 2020-01-01T00:00:00.123Z VALID 1577836800123 utc_format +iso_named_utc TEXT 2020-01-01T00:00:00.123UTC VALID 1577836800123 utc_format +iso_offset TEXT 2020-01-01T00:00:00.123+02:30 VALID 1577827800123 utc_format +iso_without_millis TEXT 2020-01-01T00:00:00Z INVALID utc_format +iso_two_millis TEXT 2020-01-01T00:00:00.12Z INVALID utc_format +iso_four_millis TEXT 2020-01-01T00:00:00.1234Z INVALID utc_format +offset_datetime TEXT 2020-01-01 00:00:00+02:00 VALID 1577829600000 numeric_zone +offset_date TEXT 2020-01-01 +02:00 VALID 1577829600000 numeric_zone +berlin_winter TEXT 2020-01-01 12:00:00Europe/Berlin VALID 1577876400000 named_zone +berlin_summer TEXT 2020-07-01 12:00:00Europe/Berlin VALID 1593597600000 named_zone +berlin_gap TEXT 2020-03-29 02:30:00Europe/Berlin VALID 1585441800000 zone_transition +berlin_overlap TEXT 2020-10-25 02:30:00Europe/Berlin VALID 1603589400000 zone_transition +hour_24 TEXT 2020-01-01 24:59:59UTC VALID 1577840399000 hour_24_wrap +hour_25 TEXT 2020-01-01 25:00:00UTC INVALID time_validation +minute_60 TEXT 2020-01-01 00:60:00UTC INVALID time_validation +second_60 TEXT 2020-01-01 00:00:60UTC INVALID time_validation +year_zero TEXT 0000-01-01 VALID -62167219200000 proleptic_year +negative_year TEXT -0001-01-01 VALID -62198755200000 proleptic_year +max_int_year TEXT 2147483647-01-01 VALID -6019000092841406464 year_wrap +min_int_year TEXT -2147483648-01-01 VALID 6018875726867006464 year_wrap +negative_year_named_zone_wrap TEXT -1000000000-01-01 00:00:00Europe/London INVALID server_unchecked_guard +year_overflow TEXT 2147483648-01-01 INVALID year_overflow +plus_year TEXT +1-01-01 INVALID strict_year +underscore_year TEXT 2_0-01-01 INVALID strict_year +bad_month TEXT 2020-13-01 INVALID calendar_validation +bad_day TEXT 2020-02-30 INVALID calendar_validation +missing_zone TEXT 2020-01-01 00:00:00 INVALID zone_required +unknown_zone TEXT 2020-01-01 Mars/Base INVALID zone_validation +trailing_date_space TEXT 2020-01-01 INVALID no_tail +trailing_zone_space TEXT 2020-01-01 UTC INVALID no_tail +raw_trailing_text TEXT 1_000x INVALID no_tail +unicode_digits UTF16_HEX ff11,ff12,ff13 INVALID ascii_only +plain_text TEXT not-a-date INVALID invalid_text diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv new file mode 100644 index 000000000..2fc76ec03 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-decimal.tsv @@ -0,0 +1,56 @@ +# case_id input target_type target_precision target_scale outcome expected_ll_hex expected_lh_hex expected_hl_hex expected_hh_hex expected_sql +8_precision_max 9.9 DECIMAL8 2 1 VALUE 0000000000000063 - - - 9.9 +8_precision_over 10.0 DECIMAL8 2 1 INVALID - - - - - +8_precision_loss 0.01 DECIMAL8 2 1 INVALID - - - - - +8_java_null DECIMAL8 2 1 NULL - - - - NULL +16_precision_max 99.99 DECIMAL16 4 2 VALUE 000000000000270f - - - 99.99 +16_precision_over 100.00 DECIMAL16 4 2 INVALID - - - - - +16_precision_loss 0.001 DECIMAL16 4 2 INVALID - - - - - +16_java_null DECIMAL16 4 2 NULL - - - - NULL +32_precision_max 999999.999 DECIMAL32 9 3 VALUE 000000003b9ac9ff - - - 999999.999 +32_precision_over 1000000.000 DECIMAL32 9 3 INVALID - - - - - +32_precision_loss 0.0001 DECIMAL32 9 3 INVALID - - - - - +32_java_null DECIMAL32 9 3 NULL - - - - NULL +64_precision_max 99999999999999999.9 DECIMAL64 18 1 VALUE 0de0b6b3a763ffff - - - 99999999999999999.9 +64_precision_over 100000000000000000.0 DECIMAL64 18 1 INVALID - - - - - +64_precision_loss 0.01 DECIMAL64 18 1 INVALID - - - - - +64_java_null DECIMAL64 18 1 NULL - - - - NULL +128_precision_max 999999999999999999.99999999999999999999 DECIMAL128 38 20 VALUE 098a223fffffffff 4b3b4ca85a86c47a - - 999999999999999999.99999999999999999999 +128_precision_over 1000000000000000000.0 DECIMAL128 38 20 INVALID - - - - - +128_precision_loss 0.000000000000000000001 DECIMAL128 38 20 INVALID - - - - - +128_java_null DECIMAL128 38 20 NULL - - - - NULL +256_precision_max 9999999999999999999.999999999999999999999999999999999999999999999999999999999 DECIMAL256 76 57 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 9999999999999999999.999999999999999999999999999999999999999999999999999999999 +256_precision_over 10000000000000000000 DECIMAL256 76 57 INVALID - - - - - +256_precision_loss 1e-58 DECIMAL256 76 57 INVALID - - - - - +256_java_null DECIMAL256 76 57 NULL - - - - NULL +8_leading_space_plus_zeros +00.5 DECIMAL8 2 1 VALUE 0000000000000005 - - - 0.5 +8_zero_exponent_after_leading_zeros 0000.e80 DECIMAL8 1 0 VALUE 0000000000000000 - - - 0 +8_full_scale_fraction .5 DECIMAL8 1 1 VALUE 0000000000000005 - - - 0.5 +8_nan NaN DECIMAL8 2 1 NULL - - - - NULL +8_empty DECIMAL8 2 1 INVALID - - - - - +16_point_without_fraction 123. DECIMAL16 4 1 VALUE 00000000000004ce - - - 123.0 +16_infinity Infinity DECIMAL16 4 1 NULL - - - - NULL +16_point_only . DECIMAL16 4 1 INVALID - - - - - +32_positive_exponent 1.23e3 DECIMAL32 9 2 VALUE 000000000001e078 - - - 1230.00 +32_positive_nan +NaN DECIMAL32 9 2 NULL - - - - NULL +32_multiple_points 1.2.3 DECIMAL32 9 2 INVALID - - - - - +64_negative_exponent -2.5e-2 DECIMAL64 18 3 VALUE ffffffffffffffe7 - - - -0.025 +64_incomplete_exponent 1e DECIMAL64 18 3 INVALID - - - - - +64_exponent_underscore 1e1_0 DECIMAL64 18 2 VALUE 000000e8d4a51000 - - - 10000000000.00 +64_nan_decimal_suffix NaNm DECIMAL64 18 3 NULL - - - - NULL +128_trailing_zeros_relaxed 1.2300 DECIMAL128 38 2 VALUE 000000000000007b 0000000000000000 - - 1.23 +64_double_suffix -7.5d DECIMAL64 18 2 VALUE fffffffffffffd12 - - - -7.50 +128_float_suffix 123.0f DECIMAL128 38 4 VALUE 000000000012c4b0 0000000000000000 - - 123.0000 +128_decimal_suffix_strict_loss 1.2300M DECIMAL128 38 2 INVALID - - - - - +128_float_then_decimal_suffix_strict_loss 1.2300fm DECIMAL128 38 2 INVALID - - - - - +128_decimal_then_float_suffix_invalid 1.2300mf DECIMAL128 38 2 INVALID - - - - - +128_nan NaN DECIMAL128 38 4 NULL - - - - NULL +128_nan_junk NaNjunk DECIMAL128 38 4 NULL - - - - NULL +256_negative_infinity -Infinity DECIMAL256 76 8 NULL - - - - NULL +256_infinity_junk Infinitygarbage DECIMAL256 76 8 NULL - - - - NULL +256_lowercase_nan nan DECIMAL256 76 8 INVALID - - - - - +256_upper_exponent 7.89E-3 DECIMAL256 76 5 VALUE 0000000000000315 0000000000000000 0000000000000000 0000000000000000 0.00789 +256_exponent_int_max 1e2147483647 DECIMAL256 76 0 INVALID - - - - - +256_exponent_int_min 1e-2147483648 DECIMAL256 76 0 INVALID - - - - - +256_full_scale_fraction 0.9999999999999999999999999999999999999999999999999999999999999999999999999999 DECIMAL256 76 76 VALUE ffffffffffffffff 7775a5f171950fff 0764b4abe8652979 161bcca7119915b5 0.9999999999999999999999999999999999999999999999999999999999999999999999999999 +256_full_scale_zero 0 DECIMAL256 76 76 VALUE 0000000000000000 0000000000000000 0000000000000000 0000000000000000 0.0000000000000000000000000000000000000000000000000000000000000000000000000000 diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv new file mode 100644 index 000000000..5f7c243bf --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-numeric.tsv @@ -0,0 +1,433 @@ +# Seven fields: case_id, input_utf16_hex, target_type, expected_wire_type, expected_wire_bits, expected_sql_value, outcome +# Input is lowercase UTF-16 code-unit hex; empty is a present empty string and is explicit null. +# Wire bits are lowercase fixed-width hexadecimal. means bitmap null/no value bytes; paired fields mean local rejection. +# BITS: means stored floating raw bits. Parsed INT/LONG minima and floating NaN have present wire bits but SQL NULL. +diag_001 BYTE BYTE 0 VALID +diag_002 BYTE BYTE INVALID +diag_003 0030 BYTE BYTE 00 0 VALID +diag_004 002d0030 BYTE BYTE 00 0 VALID +diag_005 002b0030 BYTE BYTE 00 0 VALID +diag_006 00300030 BYTE BYTE 00 0 VALID +diag_007 0031 BYTE BYTE 01 1 VALID +diag_008 002b0031 BYTE BYTE 01 1 VALID +diag_009 002d0031 BYTE BYTE ff -1 VALID +diag_010 003100320037 BYTE BYTE 7f 127 VALID +diag_011 003100320038 BYTE BYTE INVALID +diag_012 002d003100320038 BYTE BYTE 80 -128 VALID +diag_013 002d003100320039 BYTE BYTE INVALID +diag_014 00330032003700360037 BYTE BYTE INVALID +diag_015 00330032003700360038 BYTE BYTE INVALID +diag_016 002d00330032003700360038 BYTE BYTE INVALID +diag_017 002d00330032003700360039 BYTE BYTE INVALID +diag_018 0032003100340037003400380033003600340037 BYTE BYTE INVALID +diag_019 0032003100340037003400380033003600340038 BYTE BYTE INVALID +diag_020 002d0032003100340037003400380033003600340038 BYTE BYTE INVALID +diag_021 002d0032003100340037003400380033003600340039 BYTE BYTE INVALID +diag_022 0039003200320033003300370032003000330036003800350034003700370035003800300037 BYTE BYTE INVALID +diag_023 0039003200320033003300370032003000330036003800350034003700370035003800300038 BYTE BYTE INVALID +diag_024 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 BYTE BYTE INVALID +diag_025 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 BYTE BYTE INVALID +diag_026 0031002e0030 BYTE BYTE INVALID +diag_027 0031002e0035 BYTE BYTE INVALID +diag_028 003100650032 BYTE BYTE INVALID +diag_029 00310065002d0032 BYTE BYTE INVALID +diag_030 0031005f0030 BYTE BYTE 0a 10 VALID +diag_031 002d0031005f0030 BYTE BYTE f6 -10 VALID +diag_032 005f0031 BYTE BYTE INVALID +diag_033 0031005f BYTE BYTE INVALID +diag_034 0031005f005f0030 BYTE BYTE INVALID +diag_035 0031007f0030 BYTE BYTE INVALID +diag_036 0031004c BYTE BYTE INVALID +diag_037 0031006c BYTE BYTE INVALID +diag_038 00200031 BYTE BYTE INVALID +diag_039 00310020 BYTE BYTE INVALID +diag_040 00090031 BYTE BYTE INVALID +diag_041 0031000a BYTE BYTE INVALID +diag_042 0030007800310030 BYTE BYTE INVALID +diag_043 004e0061004e BYTE BYTE INVALID +diag_044 006e0061006e BYTE BYTE INVALID +diag_045 0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_046 002b0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_047 002d0049006e00660069006e006900740079 BYTE BYTE INVALID +diag_048 0069006e0066 BYTE BYTE INVALID +diag_049 00310065003300300039 BYTE BYTE INVALID +diag_050 00310065002d0034003000300030 BYTE BYTE INVALID +diag_051 00310038003400340036003700340034003000370033003700300039003500350031003600310035 BYTE BYTE INVALID +diag_052 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 BYTE BYTE INVALID +diag_053 0000 BYTE BYTE INVALID +diag_054 00000031 BYTE BYTE INVALID +diag_055 00310000 BYTE BYTE INVALID +diag_056 003100000032 BYTE BYTE INVALID +diag_057 0661 BYTE BYTE INVALID +diag_058 00a00031 BYTE BYTE INVALID +diag_059 d800 BYTE BYTE INVALID +diag_060 0031d800 BYTE BYTE INVALID +diag_061 dc00 BYTE BYTE INVALID +diag_062 SHORT SHORT 0 VALID +diag_063 SHORT SHORT INVALID +diag_064 0030 SHORT SHORT 0000 0 VALID +diag_065 002d0030 SHORT SHORT 0000 0 VALID +diag_066 002b0030 SHORT SHORT INVALID +diag_067 00300030 SHORT SHORT 0000 0 VALID +diag_068 0031 SHORT SHORT 0001 1 VALID +diag_069 002b0031 SHORT SHORT INVALID +diag_070 002d0031 SHORT SHORT ffff -1 VALID +diag_071 003100320037 SHORT SHORT 007f 127 VALID +diag_072 003100320038 SHORT SHORT 0080 128 VALID +diag_073 002d003100320038 SHORT SHORT ff80 -128 VALID +diag_074 002d003100320039 SHORT SHORT ff7f -129 VALID +diag_075 00330032003700360037 SHORT SHORT 7fff 32767 VALID +diag_076 00330032003700360038 SHORT SHORT INVALID +diag_077 002d00330032003700360038 SHORT SHORT 8000 -32768 VALID +diag_078 002d00330032003700360039 SHORT SHORT INVALID +diag_079 0032003100340037003400380033003600340037 SHORT SHORT INVALID +diag_080 0032003100340037003400380033003600340038 SHORT SHORT INVALID +diag_081 002d0032003100340037003400380033003600340038 SHORT SHORT INVALID +diag_082 002d0032003100340037003400380033003600340039 SHORT SHORT INVALID +diag_083 0039003200320033003300370032003000330036003800350034003700370035003800300037 SHORT SHORT INVALID +diag_084 0039003200320033003300370032003000330036003800350034003700370035003800300038 SHORT SHORT INVALID +diag_085 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 SHORT SHORT INVALID +diag_086 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 SHORT SHORT INVALID +diag_087 0031002e0030 SHORT SHORT INVALID +diag_088 0031002e0035 SHORT SHORT INVALID +diag_089 003100650032 SHORT SHORT INVALID +diag_090 00310065002d0032 SHORT SHORT INVALID +diag_091 0031005f0030 SHORT SHORT INVALID +diag_092 002d0031005f0030 SHORT SHORT INVALID +diag_093 005f0031 SHORT SHORT INVALID +diag_094 0031005f SHORT SHORT INVALID +diag_095 0031005f005f0030 SHORT SHORT INVALID +diag_096 0031007f0030 SHORT SHORT INVALID +diag_097 0031004c SHORT SHORT INVALID +diag_098 0031006c SHORT SHORT INVALID +diag_099 00200031 SHORT SHORT INVALID +diag_100 00310020 SHORT SHORT INVALID +diag_101 00090031 SHORT SHORT INVALID +diag_102 0031000a SHORT SHORT INVALID +diag_103 0030007800310030 SHORT SHORT INVALID +diag_104 004e0061004e SHORT SHORT INVALID +diag_105 006e0061006e SHORT SHORT INVALID +diag_106 0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_107 002b0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_108 002d0049006e00660069006e006900740079 SHORT SHORT INVALID +diag_109 0069006e0066 SHORT SHORT INVALID +diag_110 00310065003300300039 SHORT SHORT INVALID +diag_111 00310065002d0034003000300030 SHORT SHORT INVALID +diag_112 00310038003400340036003700340034003000370033003700300039003500350031003600310035 SHORT SHORT INVALID +diag_113 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 SHORT SHORT INVALID +diag_114 0000 SHORT SHORT INVALID +diag_115 00000031 SHORT SHORT INVALID +diag_116 00310000 SHORT SHORT INVALID +diag_117 003100000032 SHORT SHORT INVALID +diag_118 0661 SHORT SHORT INVALID +diag_119 00a00031 SHORT SHORT INVALID +diag_120 d800 SHORT SHORT INVALID +diag_121 0031d800 SHORT SHORT INVALID +diag_122 dc00 SHORT SHORT INVALID +diag_123 INT INT NULL VALID +diag_124 INT INT INVALID +diag_125 0030 INT INT 00000000 0 VALID +diag_126 002d0030 INT INT 00000000 0 VALID +diag_127 002b0030 INT INT 00000000 0 VALID +diag_128 00300030 INT INT 00000000 0 VALID +diag_129 0031 INT INT 00000001 1 VALID +diag_130 002b0031 INT INT 00000001 1 VALID +diag_131 002d0031 INT INT ffffffff -1 VALID +diag_132 003100320037 INT INT 0000007f 127 VALID +diag_133 003100320038 INT INT 00000080 128 VALID +diag_134 002d003100320038 INT INT ffffff80 -128 VALID +diag_135 002d003100320039 INT INT ffffff7f -129 VALID +diag_136 00330032003700360037 INT INT 00007fff 32767 VALID +diag_137 00330032003700360038 INT INT 00008000 32768 VALID +diag_138 002d00330032003700360038 INT INT ffff8000 -32768 VALID +diag_139 002d00330032003700360039 INT INT ffff7fff -32769 VALID +diag_140 0032003100340037003400380033003600340037 INT INT 7fffffff 2147483647 VALID +diag_141 0032003100340037003400380033003600340038 INT INT INVALID +diag_142 002d0032003100340037003400380033003600340038 INT INT 80000000 NULL VALID +diag_143 002d0032003100340037003400380033003600340039 INT INT INVALID +diag_144 0039003200320033003300370032003000330036003800350034003700370035003800300037 INT INT INVALID +diag_145 0039003200320033003300370032003000330036003800350034003700370035003800300038 INT INT INVALID +diag_146 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 INT INT INVALID +diag_147 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 INT INT INVALID +diag_148 0031002e0030 INT INT INVALID +diag_149 0031002e0035 INT INT INVALID +diag_150 003100650032 INT INT INVALID +diag_151 00310065002d0032 INT INT INVALID +diag_152 0031005f0030 INT INT 0000000a 10 VALID +diag_153 002d0031005f0030 INT INT fffffff6 -10 VALID +diag_154 005f0031 INT INT INVALID +diag_155 0031005f INT INT INVALID +diag_156 0031005f005f0030 INT INT INVALID +diag_157 0031007f0030 INT INT INVALID +diag_158 0031004c INT INT INVALID +diag_159 0031006c INT INT INVALID +diag_160 00200031 INT INT INVALID +diag_161 00310020 INT INT INVALID +diag_162 00090031 INT INT INVALID +diag_163 0031000a INT INT INVALID +diag_164 0030007800310030 INT INT INVALID +diag_165 004e0061004e INT INT INVALID +diag_166 006e0061006e INT INT INVALID +diag_167 0049006e00660069006e006900740079 INT INT INVALID +diag_168 002b0049006e00660069006e006900740079 INT INT INVALID +diag_169 002d0049006e00660069006e006900740079 INT INT INVALID +diag_170 0069006e0066 INT INT INVALID +diag_171 00310065003300300039 INT INT INVALID +diag_172 00310065002d0034003000300030 INT INT INVALID +diag_173 00310038003400340036003700340034003000370033003700300039003500350031003600310035 INT INT INVALID +diag_174 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 INT INT INVALID +diag_175 0000 INT INT INVALID +diag_176 00000031 INT INT INVALID +diag_177 00310000 INT INT INVALID +diag_178 003100000032 INT INT INVALID +diag_179 0661 INT INT INVALID +diag_180 00a00031 INT INT INVALID +diag_181 d800 INT INT INVALID +diag_182 0031d800 INT INT INVALID +diag_183 dc00 INT INT INVALID +diag_184 LONG LONG NULL VALID +diag_185 LONG LONG INVALID +diag_186 0030 LONG LONG 0000000000000000 0 VALID +diag_187 002d0030 LONG LONG 0000000000000000 0 VALID +diag_188 002b0030 LONG LONG INVALID +diag_189 00300030 LONG LONG 0000000000000000 0 VALID +diag_190 0031 LONG LONG 0000000000000001 1 VALID +diag_191 002b0031 LONG LONG INVALID +diag_192 002d0031 LONG LONG ffffffffffffffff -1 VALID +diag_193 003100320037 LONG LONG 000000000000007f 127 VALID +diag_194 003100320038 LONG LONG 0000000000000080 128 VALID +diag_195 002d003100320038 LONG LONG ffffffffffffff80 -128 VALID +diag_196 002d003100320039 LONG LONG ffffffffffffff7f -129 VALID +diag_197 00330032003700360037 LONG LONG 0000000000007fff 32767 VALID +diag_198 00330032003700360038 LONG LONG 0000000000008000 32768 VALID +diag_199 002d00330032003700360038 LONG LONG ffffffffffff8000 -32768 VALID +diag_200 002d00330032003700360039 LONG LONG ffffffffffff7fff -32769 VALID +diag_201 0032003100340037003400380033003600340037 LONG LONG 000000007fffffff 2147483647 VALID +diag_202 0032003100340037003400380033003600340038 LONG LONG 0000000080000000 2147483648 VALID +diag_203 002d0032003100340037003400380033003600340038 LONG LONG ffffffff80000000 -2147483648 VALID +diag_204 002d0032003100340037003400380033003600340039 LONG LONG ffffffff7fffffff -2147483649 VALID +diag_205 0039003200320033003300370032003000330036003800350034003700370035003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +diag_206 0039003200320033003300370032003000330036003800350034003700370035003800300038 LONG LONG INVALID +diag_207 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 LONG LONG 8000000000000000 NULL VALID +diag_208 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 LONG LONG INVALID +diag_209 0031002e0030 LONG LONG INVALID +diag_210 0031002e0035 LONG LONG INVALID +diag_211 003100650032 LONG LONG INVALID +diag_212 00310065002d0032 LONG LONG INVALID +diag_213 0031005f0030 LONG LONG 000000000000000a 10 VALID +diag_214 002d0031005f0030 LONG LONG fffffffffffffff6 -10 VALID +diag_215 005f0031 LONG LONG INVALID +diag_216 0031005f LONG LONG INVALID +diag_217 0031005f005f0030 LONG LONG INVALID +diag_218 0031007f0030 LONG LONG 000000000000000a 10 VALID +diag_219 0031004c LONG LONG 0000000000000001 1 VALID +diag_220 0031006c LONG LONG 0000000000000001 1 VALID +diag_221 00200031 LONG LONG INVALID +diag_222 00310020 LONG LONG INVALID +diag_223 00090031 LONG LONG INVALID +diag_224 0031000a LONG LONG INVALID +diag_225 0030007800310030 LONG LONG INVALID +diag_226 004e0061004e LONG LONG INVALID +diag_227 006e0061006e LONG LONG INVALID +diag_228 0049006e00660069006e006900740079 LONG LONG INVALID +diag_229 002b0049006e00660069006e006900740079 LONG LONG INVALID +diag_230 002d0049006e00660069006e006900740079 LONG LONG INVALID +diag_231 0069006e0066 LONG LONG INVALID +diag_232 00310065003300300039 LONG LONG INVALID +diag_233 00310065002d0034003000300030 LONG LONG INVALID +diag_234 00310038003400340036003700340034003000370033003700300039003500350031003600310035 LONG LONG INVALID +diag_235 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 LONG LONG INVALID +diag_236 0000 LONG LONG INVALID +diag_237 00000031 LONG LONG INVALID +diag_238 00310000 LONG LONG INVALID +diag_239 003100000032 LONG LONG INVALID +diag_240 0661 LONG LONG INVALID +diag_241 00a00031 LONG LONG INVALID +diag_242 d800 LONG LONG INVALID +diag_243 0031d800 LONG LONG INVALID +diag_244 dc00 LONG LONG INVALID +diag_245 FLOAT FLOAT NULL VALID +diag_246 FLOAT FLOAT INVALID +diag_247 0030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_248 002d0030 FLOAT FLOAT 80000000 BITS:80000000 VALID +diag_249 002b0030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_250 00300030 FLOAT FLOAT 00000000 BITS:00000000 VALID +diag_251 0031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_252 002b0031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_253 002d0031 FLOAT FLOAT bf800000 BITS:bf800000 VALID +diag_254 003100320037 FLOAT FLOAT 42fe0000 BITS:42fe0000 VALID +diag_255 003100320038 FLOAT FLOAT 43000000 BITS:43000000 VALID +diag_256 002d003100320038 FLOAT FLOAT c3000000 BITS:c3000000 VALID +diag_257 002d003100320039 FLOAT FLOAT c3010000 BITS:c3010000 VALID +diag_258 00330032003700360037 FLOAT FLOAT 46fffe00 BITS:46fffe00 VALID +diag_259 00330032003700360038 FLOAT FLOAT 47000000 BITS:47000000 VALID +diag_260 002d00330032003700360038 FLOAT FLOAT c7000000 BITS:c7000000 VALID +diag_261 002d00330032003700360039 FLOAT FLOAT c7000100 BITS:c7000100 VALID +diag_262 0032003100340037003400380033003600340037 FLOAT FLOAT 4f000000 BITS:4f000000 VALID +diag_263 0032003100340037003400380033003600340038 FLOAT FLOAT 4f000000 BITS:4f000000 VALID +diag_264 002d0032003100340037003400380033003600340038 FLOAT FLOAT cf000000 BITS:cf000000 VALID +diag_265 002d0032003100340037003400380033003600340039 FLOAT FLOAT cf000000 BITS:cf000000 VALID +diag_266 0039003200320033003300370032003000330036003800350034003700370035003800300037 FLOAT FLOAT 5f000000 BITS:5f000000 VALID +diag_267 0039003200320033003300370032003000330036003800350034003700370035003800300038 FLOAT FLOAT 5f000000 BITS:5f000000 VALID +diag_268 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 FLOAT FLOAT df000000 BITS:df000000 VALID +diag_269 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 FLOAT FLOAT df000000 BITS:df000000 VALID +diag_270 0031002e0030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_271 0031002e0035 FLOAT FLOAT 3fc00000 BITS:3fc00000 VALID +diag_272 003100650032 FLOAT FLOAT 42c80000 BITS:42c80000 VALID +diag_273 00310065002d0032 FLOAT FLOAT 3c23d70a BITS:3c23d70a VALID +diag_274 0031005f0030 FLOAT FLOAT INVALID +diag_275 002d0031005f0030 FLOAT FLOAT INVALID +diag_276 005f0031 FLOAT FLOAT INVALID +diag_277 0031005f FLOAT FLOAT INVALID +diag_278 0031005f005f0030 FLOAT FLOAT INVALID +diag_279 0031007f0030 FLOAT FLOAT INVALID +diag_280 0031004c FLOAT FLOAT INVALID +diag_281 0031006c FLOAT FLOAT INVALID +diag_282 00200031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_283 00310020 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_284 00090031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_285 0031000a FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_286 0030007800310030 FLOAT FLOAT INVALID +diag_287 004e0061004e FLOAT FLOAT 7fc00000 NULL VALID +diag_288 006e0061006e FLOAT FLOAT INVALID +diag_289 0049006e00660069006e006900740079 FLOAT FLOAT 7f800000 BITS:7f800000 VALID +diag_290 002b0049006e00660069006e006900740079 FLOAT FLOAT 7f800000 BITS:7f800000 VALID +diag_291 002d0049006e00660069006e006900740079 FLOAT FLOAT ff800000 BITS:ff800000 VALID +diag_292 0069006e0066 FLOAT FLOAT INVALID +diag_293 00310065003300300039 FLOAT FLOAT INVALID +diag_294 00310065002d0034003000300030 FLOAT FLOAT INVALID +diag_295 00310038003400340036003700340034003000370033003700300039003500350031003600310035 FLOAT FLOAT 5f800000 BITS:5f800000 VALID +diag_296 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 FLOAT FLOAT INVALID +diag_297 0000 FLOAT FLOAT INVALID +diag_298 00000031 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_299 00310000 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +diag_300 003100000032 FLOAT FLOAT INVALID +diag_301 0661 FLOAT FLOAT INVALID +diag_302 00a00031 FLOAT FLOAT INVALID +diag_303 d800 FLOAT FLOAT INVALID +diag_304 0031d800 FLOAT FLOAT INVALID +diag_305 dc00 FLOAT FLOAT INVALID +diag_306 DOUBLE DOUBLE NULL VALID +diag_307 DOUBLE DOUBLE INVALID +diag_308 0030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_309 002d0030 DOUBLE DOUBLE 8000000000000000 BITS:8000000000000000 VALID +diag_310 002b0030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_311 00300030 DOUBLE DOUBLE 0000000000000000 BITS:0000000000000000 VALID +diag_312 0031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_313 002b0031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_314 002d0031 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 VALID +diag_315 003100320037 DOUBLE DOUBLE 405fc00000000000 BITS:405fc00000000000 VALID +diag_316 003100320038 DOUBLE DOUBLE 4060000000000000 BITS:4060000000000000 VALID +diag_317 002d003100320038 DOUBLE DOUBLE c060000000000000 BITS:c060000000000000 VALID +diag_318 002d003100320039 DOUBLE DOUBLE c060200000000000 BITS:c060200000000000 VALID +diag_319 00330032003700360037 DOUBLE DOUBLE 40dfffc000000000 BITS:40dfffc000000000 VALID +diag_320 00330032003700360038 DOUBLE DOUBLE 40e0000000000000 BITS:40e0000000000000 VALID +diag_321 002d00330032003700360038 DOUBLE DOUBLE c0e0000000000000 BITS:c0e0000000000000 VALID +diag_322 002d00330032003700360039 DOUBLE DOUBLE c0e0002000000000 BITS:c0e0002000000000 VALID +diag_323 0032003100340037003400380033003600340037 DOUBLE DOUBLE 41dfffffffc00000 BITS:41dfffffffc00000 VALID +diag_324 0032003100340037003400380033003600340038 DOUBLE DOUBLE 41e0000000000000 BITS:41e0000000000000 VALID +diag_325 002d0032003100340037003400380033003600340038 DOUBLE DOUBLE c1e0000000000000 BITS:c1e0000000000000 VALID +diag_326 002d0032003100340037003400380033003600340039 DOUBLE DOUBLE c1e0000000200000 BITS:c1e0000000200000 VALID +diag_327 0039003200320033003300370032003000330036003800350034003700370035003800300037 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 VALID +diag_328 0039003200320033003300370032003000330036003800350034003700370035003800300038 DOUBLE DOUBLE 43e0000000000000 BITS:43e0000000000000 VALID +diag_329 002d0039003200320033003300370032003000330036003800350034003700370035003800300038 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 VALID +diag_330 002d0039003200320033003300370032003000330036003800350034003700370035003800300039 DOUBLE DOUBLE c3e0000000000000 BITS:c3e0000000000000 VALID +diag_331 0031002e0030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_332 0031002e0035 DOUBLE DOUBLE 3ff8000000000000 BITS:3ff8000000000000 VALID +diag_333 003100650032 DOUBLE DOUBLE 4059000000000000 BITS:4059000000000000 VALID +diag_334 00310065002d0032 DOUBLE DOUBLE 3f847ae147ae147b BITS:3f847ae147ae147b VALID +diag_335 0031005f0030 DOUBLE DOUBLE INVALID +diag_336 002d0031005f0030 DOUBLE DOUBLE INVALID +diag_337 005f0031 DOUBLE DOUBLE INVALID +diag_338 0031005f DOUBLE DOUBLE INVALID +diag_339 0031005f005f0030 DOUBLE DOUBLE INVALID +diag_340 0031007f0030 DOUBLE DOUBLE INVALID +diag_341 0031004c DOUBLE DOUBLE INVALID +diag_342 0031006c DOUBLE DOUBLE INVALID +diag_343 00200031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_344 00310020 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_345 00090031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_346 0031000a DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_347 0030007800310030 DOUBLE DOUBLE INVALID +diag_348 004e0061004e DOUBLE DOUBLE 7ff8000000000000 NULL VALID +diag_349 006e0061006e DOUBLE DOUBLE INVALID +diag_350 0049006e00660069006e006900740079 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 VALID +diag_351 002b0049006e00660069006e006900740079 DOUBLE DOUBLE 7ff0000000000000 BITS:7ff0000000000000 VALID +diag_352 002d0049006e00660069006e006900740079 DOUBLE DOUBLE fff0000000000000 BITS:fff0000000000000 VALID +diag_353 0069006e0066 DOUBLE DOUBLE INVALID +diag_354 00310065003300300039 DOUBLE DOUBLE INVALID +diag_355 00310065002d0034003000300030 DOUBLE DOUBLE INVALID +diag_356 00310038003400340036003700340034003000370033003700300039003500350031003600310035 DOUBLE DOUBLE 43f0000000000000 BITS:43f0000000000000 VALID +diag_357 00390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039003900390039 DOUBLE DOUBLE 4a511b0ec57e649a BITS:4a511b0ec57e649a VALID +diag_358 0000 DOUBLE DOUBLE INVALID +diag_359 00000031 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_360 00310000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +diag_361 003100000032 DOUBLE DOUBLE INVALID +diag_362 0661 DOUBLE DOUBLE INVALID +diag_363 00a00031 DOUBLE DOUBLE INVALID +diag_364 d800 DOUBLE DOUBLE INVALID +diag_365 0031d800 DOUBLE DOUBLE INVALID +diag_366 dc00 DOUBLE DOUBLE INVALID +wrap_pos_21e18 00320031003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_neg_21e18 002d00320031003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_25e18 00320035003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_42e18 00340032003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +wrap_pos_63e18 00360033003000300030003000300030003000300030003000300030003000300030003000300030 LONG LONG INVALID +overflow_u64_max 00310038003400340036003700340034003000370033003700300039003500350031003600310035 LONG LONG INVALID +overflow_u64 00310038003400340036003700340034003000370033003700300039003500350031003600310036 LONG LONG INVALID +float_hex_one 00300078003100700030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_f 003000780031007000300066 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_fraction_one 003000780031002e003000700030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_decimal_one_f 00310066 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_decimal_one_d 00310064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_hex_one 00300078003100700030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_f 003000780031007000300066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_fraction_one 003000780031002e003000700030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_decimal_one_f 00310066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_decimal_one_d 00310064 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +long_negative_suffix 002d0031004c LONG LONG ffffffffffffffff -1 VALID +long_separator_suffix 0031005f0030004c LONG LONG 000000000000000a 10 VALID +long_separator_before_suffix 0031005f004c LONG LONG INVALID +long_leading_zero_max 0030003000300039003200320033003300370032003000330036003800350034003700370035003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_max_suffix 0039003200320033003300370032003000330036003800350034003700370035003800300037006c LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_max_plus_one_suffix 0039003200320033003300370032003000330036003800350034003700370035003800300038004c LONG LONG INVALID +long_grouped_max 0039005f003200320033005f003300370032005f003000330036005f003800350034005f003700370035005f003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +long_del_grouped 0039007f003200320033007f003300370032007f003000330036007f003800350034007f003700370035007f003800300037 LONG LONG 7fffffffffffffff 9223372036854775807 VALID +float_hex_one_d 003000780031007000300064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_upper_f 003000780031007000300046 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_one_upper_d 003000780031007000300044 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_hex_upper_xp 00300058003100500030 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_hex_one_d 003000780031007000300064 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_upper_f 003000780031007000300046 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_one_upper_d 003000780031007000300044 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_hex_upper_xp 00300058003100500030 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +float_min_subnormal 0031002e00340065002d00340035 FLOAT FLOAT 00000001 BITS:00000001 VALID +float_half_subnormal_underflow 00370065002d00340036 FLOAT FLOAT INVALID +float_max_finite 0033002e0034003000320038003200330035006500330038 FLOAT FLOAT 7f7fffff BITS:7f7fffff VALID +float_above_max_finite 0033002e0034003000320038003200330036006500330038 FLOAT FLOAT INVALID +double_min_subnormal 0034002e00390065002d003300320034 DOUBLE DOUBLE 0000000000000001 BITS:0000000000000001 VALID +double_half_subnormal_underflow 0032002e00340065002d003300320034 DOUBLE DOUBLE INVALID +double_max_finite 0031002e00370039003700360039003300310033003400380036003200330031003500370065003300300038 DOUBLE DOUBLE 7fefffffffffffff BITS:7fefffffffffffff VALID +double_above_max_finite 0031002e00370039003700360039003300310033003400380036003200330031003500390065003300300038 DOUBLE DOUBLE INVALID +float_cross_suffix_trailing_space 003100640020 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_cross_suffix_internal_space 003100200064 FLOAT FLOAT INVALID +float_mixed_suffix_chain 003100660064 FLOAT FLOAT INVALID +float_nan_cross_suffix 004e0061004e0064 FLOAT FLOAT INVALID +float_infinity_cross_suffix 0049006e00660069006e0069007400790064 FLOAT FLOAT INVALID +float_negative_cross_suffix 002d00310064 FLOAT FLOAT bf800000 BITS:bf800000 VALID +float_decimal_point_cross_suffix 0031002e0064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_missing_digits_cross_suffix 002e0064 FLOAT FLOAT INVALID +float_exponent_cross_suffix 0031006500300064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_leading_nul_cross_suffix 000000310064 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +float_trailing_nul_cross_suffix 003100640000 FLOAT FLOAT 3f800000 BITS:3f800000 VALID +double_cross_suffix_trailing_space 003100660020 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_cross_suffix_internal_space 003100200066 DOUBLE DOUBLE INVALID +double_mixed_suffix_chain 003100640066 DOUBLE DOUBLE INVALID +double_nan_cross_suffix 004e0061004e0066 DOUBLE DOUBLE INVALID +double_infinity_cross_suffix 0049006e00660069006e0069007400790066 DOUBLE DOUBLE INVALID +double_negative_cross_suffix 002d00310066 DOUBLE DOUBLE bff0000000000000 BITS:bff0000000000000 VALID +double_decimal_point_cross_suffix 0031002e0066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_missing_digits_cross_suffix 002e0066 DOUBLE DOUBLE INVALID +double_exponent_cross_suffix 0031006500300066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_leading_nul_cross_suffix 000000310066 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID +double_trailing_nul_cross_suffix 003100660000 DOUBLE DOUBLE 3ff0000000000000 BITS:3ff0000000000000 VALID diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv new file mode 100644 index 000000000..f48385efc --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp-runtime.tsv @@ -0,0 +1,8 @@ +case_id target input_kind input feature +runtime_two_digit_year TIMESTAMP TEXT 24-01-02 03:04:05.006Z reference_century +runtime_greedy_negative_year TIMESTAMP TEXT -1-01-01 00:00:00.000Z greedy_year_reference +runtime_zone_gap TIMESTAMP TEXT 2024-03-31T02:30:00Europe/Berlin tzdb_gap +runtime_zone_overlap TIMESTAMP TEXT 2024-10-27T02:30:00europe/berlin tzdb_overlap_case +runtime_zone_alias TIMESTAMP TEXT 2024-01-02T03:04:05EST zone_alias +runtime_historic_zone TIMESTAMP TEXT 1880-01-01T00:00:00Europe/Berlin tzdb_history +runtime_wrapped_zone TIMESTAMP TEXT 3509450-07-01 00:00:00.000Europe/Berlin parsed_year_rules diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv new file mode 100644 index 000000000..2cf2dcf70 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/string-to-timestamp.tsv @@ -0,0 +1,111 @@ +case_id target input_kind input outcome expected_raw feature +micro_pg_millis_z TIMESTAMP TEXT 2024-01-02 03:04:05.006Z VALID 1704164645006000 format_pg +micro_fraction_1 TIMESTAMP TEXT 2024-01-02T03:04:05.1Z VALID 1704164645100000 fraction_1 +micro_fraction_2 TIMESTAMP TEXT 2024-01-02T03:04:05.12Z VALID 1704164645120000 fraction_2 +micro_fraction_3 TIMESTAMP TEXT 2024-01-02T03:04:05.123Z VALID 1704164645123000 fraction_3 +micro_fraction_4 TIMESTAMP TEXT 2024-01-02T03:04:05.1234Z VALID 1704164645234000 fraction_4_compiled_behavior +micro_fraction_6 TIMESTAMP TEXT 2024-01-02T03:04:05.123456Z VALID 1704164645123456 fraction_6 +micro_seconds_z TIMESTAMP TEXT 2024-01-02T03:04:05Z VALID 1704164645000000 format_seconds +micro_hour TIMESTAMP TEXT 2024-01-02T03 VALID 1704164400000000 format_hour +micro_day TIMESTAMP TEXT 2024-01-02 VALID 1704153600000000 format_day +micro_week TIMESTAMP TEXT 2024-W01 VALID 1704067200000000 format_week +micro_month TIMESTAMP TEXT 2024-01 VALID 1704067200000000 format_month +micro_year TIMESTAMP TEXT 2024 VALID 1704067200000000 format_year +micro_negative_year TIMESTAMP TEXT -0001-01-01 VALID -62198755200000000 signed_year +micro_hour24 TIMESTAMP TEXT 2024-01-02T24:01:00Z VALID 1704153660000000 hour24_modulo +micro_numeric_zone TIMESTAMP TEXT 2024-01-02T03:04:05+01:30 VALID 1704159245000000 numeric_zone +micro_utc_offset TIMESTAMP TEXT 2024-01-02T03:04:05UTC+01:30 VALID 1704159245000000 utc_offset +micro_epoch_negative TIMESTAMP TEXT 1969-12-31T23:59:59.999999Z VALID -1 negative +micro_epoch_zero TIMESTAMP TEXT 1970-01-01T00:00:00Z VALID 0 zero +micro_parsed_long_min TIMESTAMP TEXT -292278-01-01 00:00:00.000Z VALID -9223372036854775808 parsed_sentinel_present_sql_null +micro_null TIMESTAMP TEXT NULL null +micro_bad_day TIMESTAMP TEXT 2024-02-30 INVALID calendar +micro_bad_week TIMESTAMP TEXT 2024-W54 INVALID week +micro_bad_fraction_empty TIMESTAMP TEXT 2024-01-02T03:04:05.Z INVALID fraction +micro_bad_fraction_5 TIMESTAMP TEXT 2024-01-02T03:04:05.12345Z INVALID fraction +micro_bad_fraction_7 TIMESTAMP TEXT 2024-01-02T03:04:05.1234567Z INVALID fraction +micro_bad_plus_year TIMESTAMP TEXT +2024-01-01 INVALID signed_year +micro_signed_hour TIMESTAMP TEXT 2024-01-01T-0:00:00Z VALID 1704067200000000 signed_fixed_hour +micro_signed_minute_second TIMESTAMP TEXT 2024-01-01T00:-0:-0Z VALID 1704067200000000 signed_fixed_fields +micro_signed_month TIMESTAMP TEXT 2024-+1-01 VALID 1704067200000000 signed_fixed_month +micro_negative_fraction TIMESTAMP TEXT 2024-01-01T00:00:00.-01Z VALID 1704067199999000 signed_fraction +micro_split_negative_fraction TIMESTAMP TEXT 2024-01-01T00:00:00.123-01Z VALID 1704067200122999 signed_micro_fraction +micro_nested_negative_year TIMESTAMP TEXT --001-01-01T00:00:00Z VALID -62135596800000000 nested_fixed_year +micro_negative_week TIMESTAMP TEXT 2024-W-1 VALID 1704067200000000 signed_week +micro_int_min_year TIMESTAMP TEXT -2147483648-01-01 00:00:00.000Z VALID -9223372036854775808 year_micros_saturation +micro_malformed_high TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,d800,0031 INVALID utf16_replacement +micro_malformed_low TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,dc00,0031 INVALID utf16_replacement +micro_empty TIMESTAMP TEXT INVALID empty +micro_trailing_junk TIMESTAMP TEXT 2024-01-01T00:00:00Zjunk INVALID trailing_junk +micro_nonascii_zone_lookalike TIMESTAMP UTF16_HEX 0032,0030,0032,0034,002d,0030,0031,002d,0030,0031,0054,0030,0030,003a,0030,0030,003a,0030,0030,0045,0075,0072,006f,0070,0065,002f,0042,0065,0072,006c,0131,006e INVALID unicode_zone_lookalike +micro_fixed_year_plus TIMESTAMP TEXT +2024 INVALID fixed_year_sign +micro_fixed_year_underscore TIMESTAMP TEXT 2024_01_01 INVALID separator +micro_reachable_max TIMESTAMP TEXT 44135800-09-20 06:17:51.147Z VALID 9223372036854775800 micro_wrapped_lattice_max +micro_reachable_max_successor_wrap TIMESTAMP TEXT 44135800-09-20 06:17:51.148Z VALID -9223372036854774816 micro_unchecked_wrap +micro_nonwrapped_near_max TIMESTAMP TEXT 294247-01-10 04:00:54.775Z VALID 9223372036854775000 micro_near_max +micro_saturation_plus_millis TIMESTAMP TEXT -292278-01-01 00:00:00.001Z VALID -9223372036854774808 year_saturation_plus +micro_reachable_min_plus_eight TIMESTAMP TEXT 28937395-06-09 13:30:22.805Z VALID -9223372036854775800 micro_wrapped_lattice_min +nano_pg_millis_z TIMESTAMP_NANOS TEXT 2024-01-02 03:04:05.006Z VALID 1704164645006000000 format_pg +nano_fraction_1 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1Z VALID 1704164645100000000 fraction_1 +nano_fraction_2 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.12Z VALID 1704164645120000000 fraction_2 +nano_fraction_3 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.123Z VALID 1704164645123000000 fraction_3 +nano_fraction_4 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1234Z VALID 1704164645234000000 fraction_4_compiled_behavior +nano_fraction_6 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.123456Z VALID 1704164645123456000 fraction_6 +nano_seconds_z TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05Z VALID 1704164645000000000 format_seconds +nano_hour TIMESTAMP_NANOS TEXT 2024-01-02T03 VALID 1704164400000000000 format_hour +nano_day TIMESTAMP_NANOS TEXT 2024-01-02 VALID 1704153600000000000 format_day +nano_week TIMESTAMP_NANOS TEXT 2024-W01 VALID 1704067200000000000 format_week +nano_month TIMESTAMP_NANOS TEXT 2024-01 VALID 1704067200000000000 format_month +nano_year TIMESTAMP_NANOS TEXT 2024 VALID 1704067200000000000 format_year +nano_negative_year TIMESTAMP_NANOS TEXT -0001-01-01 INVALID nano_overflow_after_parse +nano_hour24 TIMESTAMP_NANOS TEXT 2024-01-02T24:01:00Z VALID 1704153660000000000 hour24_modulo +nano_numeric_zone TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05+01:30 VALID 1704159245000000000 numeric_zone +nano_utc_offset TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05UTC+01:30 VALID 1704159245000000000 utc_offset +nano_epoch_negative TIMESTAMP_NANOS TEXT 1969-12-31T23:59:59.999999Z VALID -1000 negative +nano_epoch_zero TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00Z VALID 0 zero +nano_parsed_long_min TIMESTAMP_NANOS TEXT -292278-01-01 00:00:00.000Z INVALID nano_overflow_after_parse +nano_null TIMESTAMP_NANOS TEXT NULL null +nano_bad_day TIMESTAMP_NANOS TEXT 2024-02-30 INVALID calendar +nano_bad_week TIMESTAMP_NANOS TEXT 2024-W54 INVALID week +nano_bad_fraction_empty TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.Z INVALID fraction +nano_bad_fraction_5 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.12345Z INVALID fraction +nano_bad_fraction_7 TIMESTAMP_NANOS TEXT 2024-01-02T03:04:05.1234567Z INVALID fraction +nano_bad_plus_year TIMESTAMP_NANOS TEXT +2024-01-01 INVALID signed_year +nano_signed_hour TIMESTAMP_NANOS TEXT 2024-01-01T-0:00:00Z VALID 1704067200000000000 signed_fixed_hour +nano_signed_minute_second TIMESTAMP_NANOS TEXT 2024-01-01T00:-0:-0Z VALID 1704067200000000000 signed_fixed_fields +nano_signed_month TIMESTAMP_NANOS TEXT 2024-+1-01 VALID 1704067200000000000 signed_fixed_month +nano_negative_fraction TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00.-01Z VALID 1704067199999000000 signed_fraction +nano_split_negative_fraction TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00.123-01Z VALID 1704067200122999000 signed_micro_fraction +nano_nested_negative_year TIMESTAMP_NANOS TEXT --001-01-01T00:00:00Z INVALID nano_overflow_after_parse +nano_negative_week TIMESTAMP_NANOS TEXT 2024-W-1 VALID 1704067200000000000 signed_week +nano_int_min_year TIMESTAMP_NANOS TEXT -2147483648-01-01 00:00:00.000Z INVALID nano_overflow_after_parse +nano_malformed_high TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,d800,0031 INVALID utf16_replacement +nano_malformed_low TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,dc00,0031 INVALID utf16_replacement +nano_empty TIMESTAMP_NANOS TEXT INVALID empty +nano_trailing_junk TIMESTAMP_NANOS TEXT 2024-01-01T00:00:00Zjunk INVALID trailing_junk +nano_nonascii_zone_lookalike TIMESTAMP_NANOS UTF16_HEX 0032,0030,0032,0034,002d,0030,0031,002d,0030,0031,0054,0030,0030,003a,0030,0030,003a,0030,0030,0045,0075,0072,006f,0070,0065,002f,0042,0065,0072,006c,0131,006e INVALID unicode_zone_lookalike +nano_fixed_year_plus TIMESTAMP_NANOS TEXT +2024 INVALID fixed_year_sign +nano_fixed_year_underscore TIMESTAMP_NANOS TEXT 2024_01_01 INVALID separator +nano_reachable_max TIMESTAMP_NANOS TEXT 44135800-09-20 06:17:51.147Z INVALID nano_overflow_after_parse +nano_reachable_max_successor_wrap TIMESTAMP_NANOS TEXT 44135800-09-20 06:17:51.148Z INVALID nano_overflow_after_parse +nano_nonwrapped_near_max TIMESTAMP_NANOS TEXT 294247-01-10 04:00:54.775Z INVALID nano_overflow_after_parse +nano_saturation_plus_millis TIMESTAMP_NANOS TEXT -292278-01-01 00:00:00.001Z INVALID nano_overflow_after_parse +nano_reachable_min_plus_eight TIMESTAMP_NANOS TEXT 28937395-06-09 13:30:22.805Z INVALID nano_overflow_after_parse +nano_upper_bound TIMESTAMP_NANOS TEXT 2262-04-11T23:47:16.854775Z VALID 9223372036854775000 nano_upper_bound +nano_upper_bound_plus_one TIMESTAMP_NANOS TEXT 2262-04-11T23:47:16.854776Z INVALID nano_overflow +nano_lower_bound TIMESTAMP_NANOS TEXT 1677-09-21T00:12:43.145225Z VALID -9223372036854775000 nano_lower_bound +nano_lower_bound_minus_one TIMESTAMP_NANOS TEXT 1677-09-21T00:12:43.145224Z INVALID nano_overflow +micro_fraction_fixed_plus TIMESTAMP TEXT 1970-01-01T00:00:00.+12Z VALID 12000 fixed_fraction_priority +micro_fraction_fixed_underscore TIMESTAMP TEXT 1970-01-01T00:00:00.1_2Z VALID 12000 fixed_fraction_priority +micro_fraction_fixed_numeric_zone TIMESTAMP TEXT 1970-01-01T00:00:00.10000 VALID 100000 fixed_fraction_priority +micro_fraction_extra_zone_tail TIMESTAMP TEXT 1970-01-01T00:00:00.10000Z INVALID fixed_fraction_priority +micro_fraction_split_signed TIMESTAMP TEXT 1970-01-01T00:00:00.1_2+34Z VALID 12034 fixed_fraction_priority +nano_fraction_fixed_plus TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.+12Z VALID 12000000 fixed_fraction_priority +nano_fraction_fixed_underscore TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.1_2Z VALID 12000000 fixed_fraction_priority +nano_fraction_fixed_numeric_zone TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.10000 VALID 100000000 fixed_fraction_priority +nano_fraction_extra_zone_tail TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.10000Z INVALID fixed_fraction_priority +nano_fraction_split_signed TIMESTAMP_NANOS TEXT 1970-01-01T00:00:00.1_2+34Z VALID 12034000 fixed_fraction_priority +micro_iso_week_retained_leap_1297 TIMESTAMP TEXT 1297-W01 VALID -21237724800000000 iso_week_source_leap +micro_iso_week_retained_leap_1697 TIMESTAMP TEXT 1697-W01 VALID -8614944000000000 iso_week_source_leap +nano_iso_week_retained_leap_1297 TIMESTAMP_NANOS TEXT 1297-W01 INVALID nano_overflow_after_parse +nano_iso_week_retained_leap_1697 TIMESTAMP_NANOS TEXT 1697-W01 VALID -8614944000000000000 iso_week_source_leap diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv new file mode 100644 index 000000000..f9f4decd0 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/text-to-text.tsv @@ -0,0 +1,91 @@ +# case_id input_type target_type input_utf16_hex expected_wire_type expected_utf8_hex expected_sql_utf16_hex +string_string_ascii STRING STRING 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +string_string_lower STRING STRING 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +string_string_controls STRING STRING 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +string_string_empty STRING STRING VARCHAR +string_string_spaces STRING STRING 002000610020 VARCHAR 206120 002000610020 +string_string_nul STRING STRING 006100000062 VARCHAR 610062 006100000062 +string_string_bmp STRING STRING 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +string_string_pair STRING STRING d83dde00 VARCHAR f09f9880 d83dde00 +string_string_high STRING STRING d800 VARCHAR 3f 003f +string_string_low STRING STRING dc00 VARCHAR 3f 003f +string_string_high_ascii STRING STRING d8000041 VARCHAR 3f41 003f0041 +string_string_high_high_low STRING STRING d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +string_string_low_high_ascii STRING STRING dc00d8000041 VARCHAR 3f3f41 003f003f0041 +string_string_question STRING STRING 003f VARCHAR 3f 003f +string_string_null STRING STRING VARCHAR +string_varchar_ascii STRING VARCHAR 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +string_varchar_lower STRING VARCHAR 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +string_varchar_controls STRING VARCHAR 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +string_varchar_empty STRING VARCHAR VARCHAR +string_varchar_spaces STRING VARCHAR 002000610020 VARCHAR 206120 002000610020 +string_varchar_nul STRING VARCHAR 006100000062 VARCHAR 610062 006100000062 +string_varchar_bmp STRING VARCHAR 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +string_varchar_pair STRING VARCHAR d83dde00 VARCHAR f09f9880 d83dde00 +string_varchar_high STRING VARCHAR d800 VARCHAR 3f 003f +string_varchar_low STRING VARCHAR dc00 VARCHAR 3f 003f +string_varchar_high_ascii STRING VARCHAR d8000041 VARCHAR 3f41 003f0041 +string_varchar_high_high_low STRING VARCHAR d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +string_varchar_low_high_ascii STRING VARCHAR dc00d8000041 VARCHAR 3f3f41 003f003f0041 +string_varchar_question STRING VARCHAR 003f VARCHAR 3f 003f +string_varchar_null STRING VARCHAR VARCHAR +string_symbol_ascii STRING SYMBOL 00510075006500730074 SYMBOL 5175657374 00510075006500730074 +string_symbol_lower STRING SYMBOL 00710075006500730074 SYMBOL 7175657374 00710075006500730074 +string_symbol_controls STRING SYMBOL 00610009000d000a0062 SYMBOL 61090d0a62 00610009000d000a0062 +string_symbol_empty STRING SYMBOL SYMBOL +string_symbol_spaces STRING SYMBOL 002000610020 SYMBOL 206120 002000610020 +string_symbol_nul STRING SYMBOL 006100000062 SYMBOL 610062 006100000062 +string_symbol_bmp STRING SYMBOL 00e94e2d SYMBOL c3a9e4b8ad 00e94e2d +string_symbol_pair STRING SYMBOL d83dde00 SYMBOL f09f9880 d83dde00 +string_symbol_high STRING SYMBOL d800 SYMBOL 3f 003f +string_symbol_low STRING SYMBOL dc00 SYMBOL 3f 003f +string_symbol_high_ascii STRING SYMBOL d8000041 SYMBOL 3f41 003f0041 +string_symbol_high_high_low STRING SYMBOL d800d83dde00 SYMBOL 3ff09f9880 003fd83dde00 +string_symbol_low_high_ascii STRING SYMBOL dc00d8000041 SYMBOL 3f3f41 003f003f0041 +string_symbol_question STRING SYMBOL 003f SYMBOL 3f 003f +string_symbol_null STRING SYMBOL SYMBOL +symbol_string_ascii SYMBOL STRING 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +symbol_string_lower SYMBOL STRING 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +symbol_string_controls SYMBOL STRING 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +symbol_string_empty SYMBOL STRING VARCHAR +symbol_string_spaces SYMBOL STRING 002000610020 VARCHAR 206120 002000610020 +symbol_string_nul SYMBOL STRING 006100000062 VARCHAR 610062 006100000062 +symbol_string_bmp SYMBOL STRING 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +symbol_string_pair SYMBOL STRING d83dde00 VARCHAR f09f9880 d83dde00 +symbol_string_high SYMBOL STRING d800 VARCHAR 3f 003f +symbol_string_low SYMBOL STRING dc00 VARCHAR 3f 003f +symbol_string_high_ascii SYMBOL STRING d8000041 VARCHAR 3f41 003f0041 +symbol_string_high_high_low SYMBOL STRING d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +symbol_string_low_high_ascii SYMBOL STRING dc00d8000041 VARCHAR 3f3f41 003f003f0041 +symbol_string_question SYMBOL STRING 003f VARCHAR 3f 003f +symbol_string_null SYMBOL STRING VARCHAR +symbol_varchar_ascii SYMBOL VARCHAR 00510075006500730074 VARCHAR 5175657374 00510075006500730074 +symbol_varchar_lower SYMBOL VARCHAR 00710075006500730074 VARCHAR 7175657374 00710075006500730074 +symbol_varchar_controls SYMBOL VARCHAR 00610009000d000a0062 VARCHAR 61090d0a62 00610009000d000a0062 +symbol_varchar_empty SYMBOL VARCHAR VARCHAR +symbol_varchar_spaces SYMBOL VARCHAR 002000610020 VARCHAR 206120 002000610020 +symbol_varchar_nul SYMBOL VARCHAR 006100000062 VARCHAR 610062 006100000062 +symbol_varchar_bmp SYMBOL VARCHAR 00e94e2d VARCHAR c3a9e4b8ad 00e94e2d +symbol_varchar_pair SYMBOL VARCHAR d83dde00 VARCHAR f09f9880 d83dde00 +symbol_varchar_high SYMBOL VARCHAR d800 VARCHAR 3f 003f +symbol_varchar_low SYMBOL VARCHAR dc00 VARCHAR 3f 003f +symbol_varchar_high_ascii SYMBOL VARCHAR d8000041 VARCHAR 3f41 003f0041 +symbol_varchar_high_high_low SYMBOL VARCHAR d800d83dde00 VARCHAR 3ff09f9880 003fd83dde00 +symbol_varchar_low_high_ascii SYMBOL VARCHAR dc00d8000041 VARCHAR 3f3f41 003f003f0041 +symbol_varchar_question SYMBOL VARCHAR 003f VARCHAR 3f 003f +symbol_varchar_null SYMBOL VARCHAR VARCHAR +symbol_symbol_ascii SYMBOL SYMBOL 00510075006500730074 SYMBOL 5175657374 00510075006500730074 +symbol_symbol_lower SYMBOL SYMBOL 00710075006500730074 SYMBOL 7175657374 00710075006500730074 +symbol_symbol_controls SYMBOL SYMBOL 00610009000d000a0062 SYMBOL 61090d0a62 00610009000d000a0062 +symbol_symbol_empty SYMBOL SYMBOL SYMBOL +symbol_symbol_spaces SYMBOL SYMBOL 002000610020 SYMBOL 206120 002000610020 +symbol_symbol_nul SYMBOL SYMBOL 006100000062 SYMBOL 610062 006100000062 +symbol_symbol_bmp SYMBOL SYMBOL 00e94e2d SYMBOL c3a9e4b8ad 00e94e2d +symbol_symbol_pair SYMBOL SYMBOL d83dde00 SYMBOL f09f9880 d83dde00 +symbol_symbol_high SYMBOL SYMBOL d800 SYMBOL 3f 003f +symbol_symbol_low SYMBOL SYMBOL dc00 SYMBOL 3f 003f +symbol_symbol_high_ascii SYMBOL SYMBOL d8000041 SYMBOL 3f41 003f0041 +symbol_symbol_high_high_low SYMBOL SYMBOL d800d83dde00 SYMBOL 3ff09f9880 003fd83dde00 +symbol_symbol_low_high_ascii SYMBOL SYMBOL dc00d8000041 SYMBOL 3f3f41 003f003f0041 +symbol_symbol_question SYMBOL SYMBOL 003f SYMBOL 3f 003f +symbol_symbol_null SYMBOL SYMBOL SYMBOL diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv new file mode 100644 index 000000000..050717e7d --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-inputs.tsv @@ -0,0 +1,103 @@ +# case_id input_kind input source_unit target_type expected_wire_value expected_sql_value +millis_micro_zero LONG 0 MILLIS TIMESTAMP 0 0 +millis_micro_neg_one LONG -1 MILLIS TIMESTAMP -1000 -1000 +millis_micro_pos_one LONG 1 MILLIS TIMESTAMP 1000 1000 +millis_micro_low_valid LONG -9223372036854775 MILLIS TIMESTAMP -9223372036854775000 -9223372036854775000 +millis_micro_low_invalid LONG -9223372036854776 MILLIS TIMESTAMP +millis_micro_high_valid LONG 9223372036854775 MILLIS TIMESTAMP 9223372036854775000 9223372036854775000 +millis_micro_high_invalid LONG 9223372036854776 MILLIS TIMESTAMP +millis_micro_long_min LONG -9223372036854775808 MILLIS TIMESTAMP +millis_nano_zero LONG 0 MILLIS TIMESTAMP_NS 0 0 +millis_nano_neg_one LONG -1 MILLIS TIMESTAMP_NS -1000000 -1000000 +millis_nano_pos_one LONG 1 MILLIS TIMESTAMP_NS 1000000 1000000 +millis_nano_low_valid LONG -9223372036854 MILLIS TIMESTAMP_NS -9223372036854000000 -9223372036854000000 +millis_nano_low_invalid LONG -9223372036855 MILLIS TIMESTAMP_NS +millis_nano_high_valid LONG 9223372036854 MILLIS TIMESTAMP_NS 9223372036854000000 9223372036854000000 +millis_nano_high_invalid LONG 9223372036855 MILLIS TIMESTAMP_NS +millis_nano_long_min LONG -9223372036854775808 MILLIS TIMESTAMP_NS +seconds_micro_zero LONG 0 SECONDS TIMESTAMP 0 0 +seconds_micro_neg_one LONG -1 SECONDS TIMESTAMP -1000000 -1000000 +seconds_micro_pos_one LONG 1 SECONDS TIMESTAMP 1000000 1000000 +seconds_micro_low_valid LONG -9223372036854 SECONDS TIMESTAMP -9223372036854000000 -9223372036854000000 +seconds_micro_low_invalid LONG -9223372036855 SECONDS TIMESTAMP +seconds_micro_high_valid LONG 9223372036854 SECONDS TIMESTAMP 9223372036854000000 9223372036854000000 +seconds_micro_high_invalid LONG 9223372036855 SECONDS TIMESTAMP +seconds_micro_long_min LONG -9223372036854775808 SECONDS TIMESTAMP +seconds_nano_zero LONG 0 SECONDS TIMESTAMP_NS 0 0 +seconds_nano_neg_one LONG -1 SECONDS TIMESTAMP_NS -1000000000 -1000000000 +seconds_nano_pos_one LONG 1 SECONDS TIMESTAMP_NS 1000000000 1000000000 +seconds_nano_low_valid LONG -9223372036 SECONDS TIMESTAMP_NS -9223372036000000000 -9223372036000000000 +seconds_nano_low_invalid LONG -9223372037 SECONDS TIMESTAMP_NS +seconds_nano_high_valid LONG 9223372036 SECONDS TIMESTAMP_NS 9223372036000000000 9223372036000000000 +seconds_nano_high_invalid LONG 9223372037 SECONDS TIMESTAMP_NS +seconds_nano_long_min LONG -9223372036854775808 SECONDS TIMESTAMP_NS +minutes_micro_zero LONG 0 MINUTES TIMESTAMP 0 0 +minutes_micro_neg_one LONG -1 MINUTES TIMESTAMP -60000000 -60000000 +minutes_micro_pos_one LONG 1 MINUTES TIMESTAMP 60000000 60000000 +minutes_micro_low_valid LONG -153722867280 MINUTES TIMESTAMP -9223372036800000000 -9223372036800000000 +minutes_micro_low_invalid LONG -153722867281 MINUTES TIMESTAMP +minutes_micro_high_valid LONG 153722867280 MINUTES TIMESTAMP 9223372036800000000 9223372036800000000 +minutes_micro_high_invalid LONG 153722867281 MINUTES TIMESTAMP +minutes_micro_long_min LONG -9223372036854775808 MINUTES TIMESTAMP +minutes_nano_zero LONG 0 MINUTES TIMESTAMP_NS 0 0 +minutes_nano_neg_one LONG -1 MINUTES TIMESTAMP_NS -60000000000 -60000000000 +minutes_nano_pos_one LONG 1 MINUTES TIMESTAMP_NS 60000000000 60000000000 +minutes_nano_low_valid LONG -153722867 MINUTES TIMESTAMP_NS -9223372020000000000 -9223372020000000000 +minutes_nano_low_invalid LONG -153722868 MINUTES TIMESTAMP_NS +minutes_nano_high_valid LONG 153722867 MINUTES TIMESTAMP_NS 9223372020000000000 9223372020000000000 +minutes_nano_high_invalid LONG 153722868 MINUTES TIMESTAMP_NS +minutes_nano_long_min LONG -9223372036854775808 MINUTES TIMESTAMP_NS +hours_micro_zero LONG 0 HOURS TIMESTAMP 0 0 +hours_micro_neg_one LONG -1 HOURS TIMESTAMP -3600000000 -3600000000 +hours_micro_pos_one LONG 1 HOURS TIMESTAMP 3600000000 3600000000 +hours_micro_low_valid LONG -2562047788 HOURS TIMESTAMP -9223372036800000000 -9223372036800000000 +hours_micro_low_invalid LONG -2562047789 HOURS TIMESTAMP +hours_micro_high_valid LONG 2562047788 HOURS TIMESTAMP 9223372036800000000 9223372036800000000 +hours_micro_high_invalid LONG 2562047789 HOURS TIMESTAMP +hours_micro_long_min LONG -9223372036854775808 HOURS TIMESTAMP +hours_nano_zero LONG 0 HOURS TIMESTAMP_NS 0 0 +hours_nano_neg_one LONG -1 HOURS TIMESTAMP_NS -3600000000000 -3600000000000 +hours_nano_pos_one LONG 1 HOURS TIMESTAMP_NS 3600000000000 3600000000000 +hours_nano_low_valid LONG -2562047 HOURS TIMESTAMP_NS -9223369200000000000 -9223369200000000000 +hours_nano_low_invalid LONG -2562048 HOURS TIMESTAMP_NS +hours_nano_high_valid LONG 2562047 HOURS TIMESTAMP_NS 9223369200000000000 9223369200000000000 +hours_nano_high_invalid LONG 2562048 HOURS TIMESTAMP_NS +hours_nano_long_min LONG -9223372036854775808 HOURS TIMESTAMP_NS +days_micro_zero LONG 0 DAYS TIMESTAMP 0 0 +days_micro_neg_one LONG -1 DAYS TIMESTAMP -86400000000 -86400000000 +days_micro_pos_one LONG 1 DAYS TIMESTAMP 86400000000 86400000000 +days_micro_low_valid LONG -106751991 DAYS TIMESTAMP -9223372022400000000 -9223372022400000000 +days_micro_low_invalid LONG -106751992 DAYS TIMESTAMP +days_micro_high_valid LONG 106751991 DAYS TIMESTAMP 9223372022400000000 9223372022400000000 +days_micro_high_invalid LONG 106751992 DAYS TIMESTAMP +days_micro_long_min LONG -9223372036854775808 DAYS TIMESTAMP +days_nano_zero LONG 0 DAYS TIMESTAMP_NS 0 0 +days_nano_neg_one LONG -1 DAYS TIMESTAMP_NS -86400000000000 -86400000000000 +days_nano_pos_one LONG 1 DAYS TIMESTAMP_NS 86400000000000 86400000000000 +days_nano_low_valid LONG -106751 DAYS TIMESTAMP_NS -9223286400000000000 -9223286400000000000 +days_nano_low_invalid LONG -106752 DAYS TIMESTAMP_NS +days_nano_high_valid LONG 106751 DAYS TIMESTAMP_NS 9223286400000000000 9223286400000000000 +days_nano_high_invalid LONG 106752 DAYS TIMESTAMP_NS +days_nano_long_min LONG -9223372036854775808 DAYS TIMESTAMP_NS +instant_micro_zero INSTANT 1970-01-01T00:00:00Z - TIMESTAMP 0 0 +instant_micro_pos_submicro INSTANT 1970-01-01T00:00:00.000000999Z - TIMESTAMP 0 0 +instant_micro_neg_submicro INSTANT 1969-12-31T23:59:59.999999999Z - TIMESTAMP -1 -1 +instant_micro_min INSTANT -290308-12-21T19:59:05.224192Z - TIMESTAMP -9223372036854775808 NULL +instant_micro_min_fraction INSTANT -290308-12-21T19:59:05.224192999Z - TIMESTAMP -9223372036854775808 NULL +instant_micro_min_plus_one INSTANT -290308-12-21T19:59:05.224193Z - TIMESTAMP -9223372036854775807 -9223372036854775807 +instant_micro_below_min INSTANT -290308-12-21T19:59:05.224191999Z - TIMESTAMP +instant_micro_max INSTANT +294247-01-10T04:00:54.775807Z - TIMESTAMP 9223372036854775807 9223372036854775807 +instant_micro_max_fraction INSTANT +294247-01-10T04:00:54.775807999Z - TIMESTAMP 9223372036854775807 9223372036854775807 +instant_micro_above_max INSTANT +294247-01-10T04:00:54.775808Z - TIMESTAMP +instant_nano_zero INSTANT 1970-01-01T00:00:00Z - TIMESTAMP_NS 0 0 +instant_nano_pos_submicro INSTANT 1970-01-01T00:00:00.000000999Z - TIMESTAMP_NS 999 999 +instant_nano_neg_submicro INSTANT 1969-12-31T23:59:59.999999999Z - TIMESTAMP_NS -1 -1 +instant_nano_min INSTANT 1677-09-21T00:12:43.145224192Z - TIMESTAMP_NS -9223372036854775808 NULL +instant_nano_min_plus_one INSTANT 1677-09-21T00:12:43.145224193Z - TIMESTAMP_NS -9223372036854775807 -9223372036854775807 +instant_nano_below_min INSTANT 1677-09-21T00:12:43.145224191Z - TIMESTAMP_NS +instant_nano_max INSTANT 2262-04-11T23:47:16.854775807Z - TIMESTAMP_NS 9223372036854775807 9223372036854775807 +instant_nano_above_max INSTANT 2262-04-11T23:47:16.854775808Z - TIMESTAMP_NS +instant_min_micro INSTANT -1000000000-01-01T00:00:00Z - TIMESTAMP +instant_max_micro INSTANT +1000000000-12-31T23:59:59.999999999Z - TIMESTAMP +instant_min_nano INSTANT -1000000000-01-01T00:00:00Z - TIMESTAMP_NS +instant_max_nano INSTANT +1000000000-12-31T23:59:59.999999999Z - TIMESTAMP_NS diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv new file mode 100644 index 000000000..69ceaa4bd --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-to-text.tsv @@ -0,0 +1,64 @@ +# case_id input_kind unit value nanos outcome expected_text +micro_min LONG MICROS -9223372036854775808 VALUE +micro_min_plus_one LONG MICROS -9223372036854775807 VALUE -290308-01-01T19:59:05.224Z +micro_minus_1000 LONG MICROS -1000 VALUE 1969-12-31T23:59:59.999Z +micro_minus_999 LONG MICROS -999 VALUE 1969-12-31T23:59:59.999Z +micro_minus_one LONG MICROS -1 VALUE 1969-12-31T23:59:59.999Z +micro_zero LONG MICROS 0 VALUE 1970-01-01T00:00:00.000Z +micro_one LONG MICROS 1 VALUE 1970-01-01T00:00:00.000Z +micro_999 LONG MICROS 999 VALUE 1970-01-01T00:00:00.000Z +micro_1000 LONG MICROS 1000 VALUE 1970-01-01T00:00:00.001Z +micro_max LONG MICROS 9223372036854775807 VALUE 294247-01-10T04:00:54.775Z +micro_year_minus_one LONG MICROS -62198755200000000 VALUE 000-1-01-01T00:00:00.000Z +micro_year_zero LONG MICROS -62167219200000000 VALUE 0001-01-01T00:00:00.000Z +micro_year_one LONG MICROS -62135596800000000 VALUE 0001-01-01T00:00:00.000Z +micro_year_99 LONG MICROS -59042995200000000 VALUE 0099-01-01T00:00:00.000Z +micro_year_100 LONG MICROS -59011459200000000 VALUE 0100-01-01T00:00:00.000Z +micro_year_9999 LONG MICROS 253370764800000000 VALUE 9999-01-01T00:00:00.000Z +micro_year_10000 LONG MICROS 253402300800000000 VALUE 10000-01-01T00:00:00.000Z +nano_min LONG NANOS -9223372036854775808 VALUE 1677-09-21T00:12:43.145Z +nano_min_plus_one LONG NANOS -9223372036854775807 VALUE 1677-09-21T00:12:43.145Z +nano_minus_1000 LONG NANOS -1000 VALUE 1969-12-31T23:59:59.999Z +nano_minus_999 LONG NANOS -999 VALUE 1970-01-01T00:00:00.000Z +nano_minus_one LONG NANOS -1 VALUE 1970-01-01T00:00:00.000Z +nano_zero LONG NANOS 0 VALUE 1970-01-01T00:00:00.000Z +nano_one LONG NANOS 1 VALUE 1970-01-01T00:00:00.000Z +nano_999 LONG NANOS 999 VALUE 1970-01-01T00:00:00.000Z +nano_1000 LONG NANOS 1000 VALUE 1970-01-01T00:00:00.000Z +nano_max LONG NANOS 9223372036854775807 VALUE 2262-04-11T23:47:16.854Z +millis_negative LONG MILLIS -1 VALUE 1969-12-31T23:59:59.999Z +millis_positive LONG MILLIS 1 VALUE 1970-01-01T00:00:00.001Z +seconds_negative LONG SECONDS -1 VALUE 1969-12-31T23:59:59.000Z +seconds_positive LONG SECONDS 1 VALUE 1970-01-01T00:00:01.000Z +minutes_positive LONG MINUTES 1 VALUE 1970-01-01T00:01:00.000Z +hours_negative LONG HOURS -1 VALUE 1969-12-31T23:00:00.000Z +days_positive LONG DAYS 1 VALUE 1970-01-02T00:00:00.000Z +days_overflow LONG DAYS 9223372036854775807 INVALID +millis_max_accepted LONG MILLIS 9223372036854775 VALUE 294247-01-10T04:00:54.775Z +millis_positive_overflow LONG MILLIS 9223372036854776 INVALID +millis_min_accepted LONG MILLIS -9223372036854775 VALUE -290308-01-01T19:59:05.225Z +millis_negative_overflow LONG MILLIS -9223372036854776 INVALID +seconds_max_accepted LONG SECONDS 9223372036854 VALUE 294247-01-10T04:00:54.000Z +seconds_positive_overflow LONG SECONDS 9223372036855 INVALID +seconds_min_accepted LONG SECONDS -9223372036854 VALUE -290308-01-01T19:59:06.000Z +seconds_negative_overflow LONG SECONDS -9223372036855 INVALID +minutes_max_accepted LONG MINUTES 153722867280 VALUE 294247-01-10T04:00:00.000Z +minutes_positive_overflow LONG MINUTES 153722867281 INVALID +minutes_min_accepted LONG MINUTES -153722867280 VALUE -290308-01-01T20:00:00.000Z +minutes_negative_overflow LONG MINUTES -153722867281 INVALID +hours_max_accepted LONG HOURS 2562047788 VALUE 294247-01-10T04:00:00.000Z +hours_positive_overflow LONG HOURS 2562047789 INVALID +hours_min_accepted LONG HOURS -2562047788 VALUE -290308-01-01T20:00:00.000Z +hours_negative_overflow LONG HOURS -2562047789 INVALID +days_max_accepted LONG DAYS 106751991 VALUE 294247-01-10T00:00:00.000Z +days_positive_overflow LONG DAYS 106751992 INVALID +days_min_accepted LONG DAYS -106751991 VALUE -290308-01-01T00:00:00.000Z +days_negative_overflow LONG DAYS -106751992 INVALID +instant_negative_fraction INSTANT -1 999999999 VALUE 1969-12-31T23:59:59.999Z +instant_epoch INSTANT 0 0 VALUE 1970-01-01T00:00:00.000Z +instant_micro_max INSTANT 9223372036854 775807000 VALUE 294247-01-10T04:00:54.775Z +instant_micro_min INSTANT -9223372036855 224192000 VALUE +instant_above_micro_max INSTANT 9223372036854 775808000 INVALID +instant_below_micro_min INSTANT -9223372036855 224191000 INVALID +instant_micro_min_plus_one INSTANT -9223372036855 224193000 VALUE -290308-01-01T19:59:05.224Z +instant_micro_max_fraction INSTANT 9223372036854 775807999 VALUE 294247-01-10T04:00:54.775Z diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv new file mode 100644 index 000000000..f55c4c584 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/timestamp-units.tsv @@ -0,0 +1,49 @@ +# case_id input source_unit target_type expected_wire_value expected_sql_value +micro_micro_zero 0 MICROS TIMESTAMP 0 0 +micro_micro_neg_1001 -1001 MICROS TIMESTAMP -1001 -1001 +micro_micro_neg_1000 -1000 MICROS TIMESTAMP -1000 -1000 +micro_micro_neg_999 -999 MICROS TIMESTAMP -999 -999 +micro_micro_neg_one -1 MICROS TIMESTAMP -1 -1 +micro_micro_pos_one 1 MICROS TIMESTAMP 1 1 +micro_micro_pos_999 999 MICROS TIMESTAMP 999 999 +micro_micro_pos_1000 1000 MICROS TIMESTAMP 1000 1000 +micro_micro_pos_1001 1001 MICROS TIMESTAMP 1001 1001 +micro_micro_min -9223372036854775808 MICROS TIMESTAMP -9223372036854775808 NULL +micro_micro_max 9223372036854775807 MICROS TIMESTAMP 9223372036854775807 9223372036854775807 +nano_nano_zero 0 NANOS TIMESTAMP_NS 0 0 +nano_nano_neg_1001 -1001 NANOS TIMESTAMP_NS -1001 -1001 +nano_nano_neg_1000 -1000 NANOS TIMESTAMP_NS -1000 -1000 +nano_nano_neg_999 -999 NANOS TIMESTAMP_NS -999 -999 +nano_nano_neg_one -1 NANOS TIMESTAMP_NS -1 -1 +nano_nano_pos_one 1 NANOS TIMESTAMP_NS 1 1 +nano_nano_pos_999 999 NANOS TIMESTAMP_NS 999 999 +nano_nano_pos_1000 1000 NANOS TIMESTAMP_NS 1000 1000 +nano_nano_pos_1001 1001 NANOS TIMESTAMP_NS 1001 1001 +nano_nano_min -9223372036854775808 NANOS TIMESTAMP_NS -9223372036854775808 NULL +nano_nano_max 9223372036854775807 NANOS TIMESTAMP_NS 9223372036854775807 9223372036854775807 +nano_micro_pos_fraction 999 NANOS TIMESTAMP 0 0 +nano_micro_zero 0 NANOS TIMESTAMP 0 0 +nano_micro_pos_one 1 NANOS TIMESTAMP 0 0 +nano_micro_pos_exact 1000 NANOS TIMESTAMP 1 1 +nano_micro_pos_over 1001 NANOS TIMESTAMP 1 1 +nano_micro_neg_fraction -999 NANOS TIMESTAMP 0 0 +nano_micro_neg_one -1 NANOS TIMESTAMP 0 0 +nano_micro_neg_exact -1000 NANOS TIMESTAMP -1 -1 +nano_micro_neg_over -1001 NANOS TIMESTAMP -1 -1 +nano_micro_min -9223372036854775808 NANOS TIMESTAMP -9223372036854775 -9223372036854775 +nano_micro_max 9223372036854775807 NANOS TIMESTAMP 9223372036854775 9223372036854775 +micro_nano_min_valid -9223372036854775 MICROS TIMESTAMP_NS -9223372036854775000 -9223372036854775000 +micro_nano_zero 0 MICROS TIMESTAMP_NS 0 0 +micro_nano_neg_one -1 MICROS TIMESTAMP_NS -1000 -1000 +micro_nano_neg_999 -999 MICROS TIMESTAMP_NS -999000 -999000 +micro_nano_neg_1000 -1000 MICROS TIMESTAMP_NS -1000000 -1000000 +micro_nano_neg_1001 -1001 MICROS TIMESTAMP_NS -1001000 -1001000 +micro_nano_below_min -9223372036854776 MICROS TIMESTAMP_NS +micro_nano_max_valid 9223372036854775 MICROS TIMESTAMP_NS 9223372036854775000 9223372036854775000 +micro_nano_pos_one 1 MICROS TIMESTAMP_NS 1000 1000 +micro_nano_pos_999 999 MICROS TIMESTAMP_NS 999000 999000 +micro_nano_pos_1000 1000 MICROS TIMESTAMP_NS 1000000 1000000 +micro_nano_pos_1001 1001 MICROS TIMESTAMP_NS 1001000 1001000 +micro_nano_above_max 9223372036854776 MICROS TIMESTAMP_NS +micro_nano_min -9223372036854775808 MICROS TIMESTAMP_NS +micro_nano_max 9223372036854775807 MICROS TIMESTAMP_NS diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv new file mode 100644 index 000000000..789bd6277 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv @@ -0,0 +1,21 @@ +# case_id input expected +null +empty +nil 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 +all_fs ffffffff-ffff-ffff-ffff-ffffffffffff ffffffff-ffff-ffff-ffff-ffffffffffff +upper_mixed 01234567-89AB-cDeF-0123-456789aBCDef 01234567-89ab-cdef-0123-456789abcdef +distinct_limbs 01234567-89ab-cdef-fedc-ba9876543210 01234567-89ab-cdef-fedc-ba9876543210 +both_min_null 80000000-0000-0000-8000-000000000000 +high_min_only 80000000-0000-0000-0000-000000000000 80000000-0000-0000-0000-000000000000 +low_min_only 00000000-0000-0000-8000-000000000000 00000000-0000-0000-8000-000000000000 +bad_hex_high_first g1234567-89ab-cdef-0123-456789abcdef +bad_hex_high_second 01234567-x9ab-cdef-0123-456789abcdef +bad_hex_high_third 01234567-89ab-xdef-0123-456789abcdef +bad_hex_low_first 01234567-89ab-cdef-x123-456789abcdef +bad_hex_low_second 01234567-89ab-cdef-0123-x56789abcdef +short_java_uuid 1-1-1-1-1 +leading_space 01234567-89ab-cdef-0123-456789abcdef +trailing_space 01234567-89ab-cdef-0123-456789abcdef +unicode_hex ０1234567-89ab-cdef-0123-456789abcdef +wrong_dash 01234567_89ab-cdef-0123-456789abcdef +missing_dash 0123456789ab-cdef-0123-456789abcdef diff --git a/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv new file mode 100644 index 000000000..4ad4ad323 --- /dev/null +++ b/core/src/test/resources/io/questdb/client/cutlass/qwp/uuid-to-text.tsv @@ -0,0 +1,17 @@ +# case_id input_lo input_hi target_type expected_wire_type expected_utf8_hex expected_sql_value +uuid_string_asym -4941174770667877871 -6850330615249940744 STRING VARCHAR 61306565626339392d396330622d346566382d626236642d366262396264333830613131 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +uuid_string_zero 0 0 STRING VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303030 00000000-0000-0000-0000-000000000000 +uuid_string_one 1 0 STRING VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303031 00000000-0000-0000-0000-000000000001 +uuid_string_ones -1 -1 STRING VARCHAR 66666666666666662d666666662d666666662d666666662d666666666666666666666666 ffffffff-ffff-ffff-ffff-ffffffffffff +uuid_string_lo_min -9223372036854775808 0 STRING VARCHAR 30303030303030302d303030302d303030302d383030302d303030303030303030303030 00000000-0000-0000-8000-000000000000 +uuid_string_hi_min 0 -9223372036854775808 STRING VARCHAR 38303030303030302d303030302d303030302d303030302d303030303030303030303030 80000000-0000-0000-0000-000000000000 +uuid_string_both_min -9223372036854775808 -9223372036854775808 STRING VARCHAR +uuid_string_mixed 81985529216486895 -81985529216486896 STRING VARCHAR 66656463626139382d373635342d333231302d303132332d343536373839616263646566 fedcba98-7654-3210-0123-456789abcdef +uuid_varchar_asym -4941174770667877871 -6850330615249940744 VARCHAR VARCHAR 61306565626339392d396330622d346566382d626236642d366262396264333830613131 a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 +uuid_varchar_zero 0 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303030 00000000-0000-0000-0000-000000000000 +uuid_varchar_one 1 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d303030302d303030303030303030303031 00000000-0000-0000-0000-000000000001 +uuid_varchar_ones -1 -1 VARCHAR VARCHAR 66666666666666662d666666662d666666662d666666662d666666666666666666666666 ffffffff-ffff-ffff-ffff-ffffffffffff +uuid_varchar_lo_min -9223372036854775808 0 VARCHAR VARCHAR 30303030303030302d303030302d303030302d383030302d303030303030303030303030 00000000-0000-0000-8000-000000000000 +uuid_varchar_hi_min 0 -9223372036854775808 VARCHAR VARCHAR 38303030303030302d303030302d303030302d303030302d303030303030303030303030 80000000-0000-0000-0000-000000000000 +uuid_varchar_both_min -9223372036854775808 -9223372036854775808 VARCHAR VARCHAR +uuid_varchar_mixed 81985529216486895 -81985529216486896 VARCHAR VARCHAR 66656463626139382d373635342d333231302d303132332d343536373839616263646566 fedcba98-7654-3210-0123-456789abcdef diff --git a/design/schema-aware-sender-contract-simplification.md b/design/schema-aware-sender-contract-simplification.md new file mode 100644 index 000000000..8ba8b03e1 --- /dev/null +++ b/design/schema-aware-sender-contract-simplification.md @@ -0,0 +1,221 @@ +# Schema-aware Java sender: contract simplification proposals + +Date: 2026-09-18 + +Status: proposal 1 is implemented (see the "One schema snapshot per table per batch" section of +[schema-aware-sender.md](schema-aware-sender.md)), with one deviation: there is no explicit refresh +operation. Batch-boundary adoption relies on ACK schema feedback and reconnects, because a caller cannot +distinguish a stale-snapshot rejection from invalid input and so has no sound rule for when to refresh. +Proposals 2 to 4 remain proposals for discussion, not approved contract or implemented behavior. + +These ideas come from the second complexity review of client +`7a73e58e8d9354a045c9895c563e8009cd6ff8cf`, including the corresponding server paths inspected at +`6b0c992b18`. They deliberately question the contract rather than only refactoring its implementation. + +## Recommendation + +Simplify the promises before doing another mechanical cleanup. Much of the remaining complexity implements +server-compatible casting, automatic schema refresh, and adaptation while batches are being built. Those promises +are stronger than the core requirement: + +> Validate types before creating, and potentially persisting, a data frame. + +The proposed direction is typed writes checked against a stable schema snapshot, followed by the existing encoder +and persistence pipeline. Do not build a general conversion framework or another buffering lifecycle. + +Start with proposal 1: extend the batch-boundary rule to schema versions and make refresh explicit. It offers a +substantial lifecycle simplification without first redesigning the conversion catalogue or wire protocol. + +Validation remains relative to the chosen snapshot. It cannot guarantee eventual server acceptance after concurrent +DDL, permission changes, or other server-side failures. The current implementation does not provide that guarantee +either. + +## 1. One schema snapshot per table per batch + +### Assumption to change + +Every new row should observe the latest cached schema, and a failed setter should automatically refresh metadata +before deciding which error to report. + +Currently, a schema identity change can replace a table buffer and retain its previous generation even when the +relevant column is unchanged. This requires pending generations, additional flush collection and accounting, and +cleanup across reset, flush, and close. The one-refresh policy also tracks whether each failing operation already +obtained fresh metadata and compares its old and new targets. + +### Proposed contract + +- Pin one schema snapshot per table for the entire pending batch. +- Adopt another snapshot only after successful flush or explicit reset. A failed flush retains the existing snapshot. +- Make explicit refresh an empty-batch operation. Initial lookup and lookup on a cache miss still obtain metadata. +- Report local validation failures directly; do not perform a network lookup from the failed setter. +- Do not implicitly flush pending rows merely to adopt a new schema. + +This is separate from the already-resolved legacy-to-modern transition. It applies the same boundary principle to +different schema versions within schema mode. + +### Complexity that can disappear + +`retiredTableBuffers`, multiple pending generations of one table, their flush/accounting/cleanup paths, per-operation +freshness bookkeeping, relevant-target comparison, and the automatic `SCHEMA_CHANGED` recovery path. + +Layout replacement is still necessary when a schema changes: `QwpTableBuffer.reset()` retains column definitions. +The simplification is that replacement happens when the old buffer has no pending rows, rather than retaining both +layouts for a later flush. + +### User impact + +Applications must coordinate schema changes and refresh deliberately. An input made valid by DDL may continue to +be rejected until refresh. Earlier completed rows remain intact; refresh does not silently discard or rewrite them. + +Evidence: [QwpWebSocketSender](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java), +especially `bindingForEffectiveWrite()`, `installSchemaBinding()`, `refreshAfterSchemaRejection()`, and +`collectNonEmptyTables()`. The existing +[`testUnrelatedVersionChangeKeepsOriginalRejectionAndRetainsOldRows()`](../core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java) +demonstrates separate generations after an unrelated version change. + +## 2. Keep schema identities out of new persisted data frames + +### Assumption to change + +Local schema validation requires a schema-extended data-frame format and schema feedback in write responses. + +The current `(tableId, metadataVersion)` pair is not an acceptance precondition. The server processes wire values +normally; matching identities suppress schema feedback. A version mismatch alone does not reject the write or +prevent the server from applying another conversion. + +### Proposed contract + +Retain schema lookup and local validation, then emit ordinary QWP data frames. Replace automatic ACK/NACK schema +feedback with explicit refresh. Keep the validation snapshot in memory; replay does not need to reconstruct it. + +This requires a coordinated client/server change, not merely ignoring feedback in the client. Schema lookup still +needs capability negotiation and authorization. + +### Complexity that can disappear + +- Schema identities and the additional data-frame encoding path. +- Schema-update lists and invalidate-all responses, including their ordering and size-limit rules. +- Schema-specific persisted-frame detection and replay capability restrictions for the new ordinary frames. + +### User impact and migration constraints + +Applications lose passive schema updates from writes and must refresh deliberately. Ordinary ACK, durable ACK, +terminal-NACK, and replay semantics must remain unchanged. + +Do not equate ordinary framing with reverting all value encoding. In particular, preserve the forced GEOHASH null +bitmap needed to distinguish valid all-one values from sentinel NULL. Calling today's legacy encoder unchanged is +not an equivalent implementation. + +Any already-persisted extended frames must retain a compatible reader/drainer until they are drained. Do not strip +identities, rewrite queued frames, discard data, or advance watermarks to simplify migration. That compatibility +requirement may delay deletion of old replay support. + +Evidence: server +[`QwpIngressProcessorState.captureSchemaFeedback()`](../../core/src/main/java/io/questdb/cutlass/qwp/server/QwpIngressProcessorState.java); +client [QwpWebSocketEncoder](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java), +[WebSocketResponse](../core/src/main/java/io/questdb/client/cutlass/qwp/client/WebSocketResponse.java), and +[`QwpColumnWriter.encodeSchemaTable()`](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpColumnWriter.java). + +## 3. Validate typed inputs instead of reproducing the server's casting catalogue + +### Assumption to change + +The client must accept essentially every intentional setter-to-column conversion that the server accepts, with +compatible parsing, formatting, rounding, and null behavior. + +`stringColumn()` currently supports numeric parsing, timestamps, dates, UUIDs, decimals, geohashes, binary, and +other targets. Other setters implement reverse formatting conversions. This makes the binding a conversion engine, +not merely a validator. + +### Proposed contract + +Accept native typed inputs and a small, explicitly documented set of safe conversions. Reject arbitrary cross-family +casts before framing. For example: + +- UUID columns require UUID input rather than UUID-looking strings. +- Timestamp columns use timestamp setters. +- Applications format numbers themselves when writing text columns. +- DOUBLE arrays remain the only supported array element type; validate the rank against the target. + +Keep value-dependent validation where type compatibility alone is insufficient: decimal precision and scale, +timestamp overflow, numeric range where conversions remain, and array shape/rank constraints. Narrowing the cast +catalogue must not defer these checks to the server. + +### Complexity that can disappear + +Temporal text conversion alone accounts for the dedicated 818-line parser and 187-line formatter at the reviewed +revision. Removing those conversions makes both files unnecessary. A narrower catalogue also removes many binding +switch branches, formatting/parsing helpers, scratch sinks, and compatibility-specific test cases. + +Do not delete shared utilities merely because one consumer disappears. The existing typed APIs may still need +parsing or representation normalization; the target is implicit cross-family casting, not all conversion code. + +### User impact + +Some currently accepted setter/column combinations become local errors. Applications must parse or format explicitly +and choose the appropriate setter. DATE needs a deliberate mapping because `Sender` has no `dateColumn()` method. + +This is the largest compatibility change, but also the largest reduction in conversion code. Agree on the retained +conversion list before implementation rather than growing it opportunistically during the rewrite. + +Evidence: [QwpSchemaBinding](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java), +[QwpSchemaTimestampParser](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampParser.java), +and [QwpSchemaTimestampFormatter](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaTimestampFormatter.java). + +## 4. Require a known target schema for validated writes + +### Assumption to change + +The same validated path must also support old servers, fresh-process offline ingestion, and automatic table/column +creation without known target metadata. + +The current asynchronous cold-start path deliberately accepts legacy writes without metadata. It cannot satisfy an +unconditional validate-before-framing guarantee. A confirmed missing table or column likewise has no existing target +type to validate against; inference is a separate contract. + +### Proposed contract + +- Cold-start writes wait or fail until metadata is available. +- Cached schemas may still support offline buffering. +- Unknown tables and columns require provisioning and refresh instead of inference and automatic creation. +- If old-server compatibility remains, expose it as explicitly unchecked behavior outside the validated contract, + not as an automatic fallback within it. + +### Complexity that can disappear + +Missing-schema inference and adoption, unknown target identities, and opportunistic legacy-to-schema transitions +within the validated path. + +### User impact + +Less zero-setup ingestion, no fresh-process offline ingestion without metadata, and explicit schema provisioning. +Do not compensate automatically by adding a persisted metadata cache: that introduces another format and freshness +lifecycle that needs its own justification. + +Evidence: [`QwpWebSocketSender.bindingForEffectiveWrite()`](../core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java), +the missing-target branches in [QwpSchemaBinding](../core/src/main/java/io/questdb/client/cutlass/qwp/protocol/QwpSchemaBinding.java), +and `testAsyncWritesLegacyOfflineThenUsesSchemaAfterSupportingConnect()` in +[QwpSchemaSenderIntegrationTest](../core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpSchemaSenderIntegrationTest.java). + +## Validation timing and complexity worth keeping + +The core requirement permits type errors at `at()`/`atNow()` rather than at every setter. Allow that flexibility if it +lets validation reuse the existing row-commit and rollback machinery. Do not introduce a second store of original +inputs merely to delay validation. + +Avoid moving all validation to flush: recovering one rejected row is substantially easier than reconstructing a +rejected batch. Every encoding path, including auto-flush and close, must still be protected before it creates a +data frame. + +Keep immutable snapshots, bounded lookup with cancellation, authorization checks, row rollback, parameter/range +validation, and unchanged-byte replay. Those mechanisms support the actual requirement or existing delivery safety. + +## Review validation and limits + +The review ran 252 focused client tests on Java 8 at the reviewed revision; all passed. That validates the current +contract, not these proposed replacements. The proposals were derived from source and contract analysis, not an +implemented prototype or measured performance comparison. + +Each adopted contract change needs explicit regression coverage for its new acceptance/error behavior, pending-row +retention, auto-flush, transaction boundaries, and replay compatibility as applicable. No production implementation +was changed as part of this review or this document. diff --git a/design/schema-aware-sender-journal.md b/design/schema-aware-sender-journal.md new file mode 100644 index 000000000..3f3421b29 --- /dev/null +++ b/design/schema-aware-sender-journal.md @@ -0,0 +1,8075 @@ +# Schema-aware sender implementation journal + +North star: [schema-aware sender design](schema-aware-sender.md), revision 59 +(revision 9 at kickoff; source-backed clarifications recorded below). + +This journal records decisions, evidence, review outcomes and remaining work. +The design defines the intended product contract; this journal must not imply +that an unimplemented part already works. Corrections are recorded explicitly. + +Current status: iterations 1 through 2.37 are accepted for their recorded scopes, including typed +microsecond/nanosecond conversion (D020), broader timestamp units and `Instant` +with the approved range/precision rules (D021), and text/symbol conversions +(D023). Boolean conversion and strict ASCII text-to-boolean parsing are accepted +in D024. D030 revises D022's sequencing: public Sender integration is accepted +on the unreleased branch; full conversion coverage still gates release. +FLOAT/DOUBLE input conversion is accepted under the approved D026 rules, +including consistent NaN missing values and mathematical integer range checks. +STRING-to-numeric conversion is accepted in D027, preserving target-specific +syntax and rejecting LONG text overflow under the approved exception. +BINARY targets and exception-safe string/binary append rollback are accepted +in D029. LONG/UUID/floating text output is accepted in D039–D041; LONG and +STRING input to ordinary timestamps is accepted in D042–D043. Timestamp-to-text +with checked normalization is accepted in D044. Native CHAR and STRING-to-CHAR +are accepted in D045; STRING-to-LONG256 is accepted in D046, STRING-to-GEOHASH +in D047, and all 18 logical native decimal-to-decimal pairs in D048. +LONG-to-decimal is accepted in D049, STRING-to-decimal in D050, +FLOAT/DOUBLE-to-decimal in D051 and native-decimal-to-text in D053. +Negotiated discovery, lookup/cache coordination, ACK/NACK feedback, pinned +schema framing and safe SF recovery/replay are implemented and tested. +Non-owning bindings provide local UUID, LONG-to-numeric, primitive/`Instant` +timestamp, text/symbol, boolean, FLOAT/DOUBLE, STRING numeric, BINARY-target and +native decimal-to-decimal, LONG/STRING/FLOAT/DOUBLE-to-decimal and native- +decimal-to-text conversion over caller-owned table buffers. Iteration 2.38 adds +the textual decimal overload to all decimal and text targets and is locally +validated but not committed. +Review corrections are recorded in D016 and D019; the approved LONG null rule +is recorded in D013. + +Automatic row-boundary schema adoption and ordinary Sender conversion are +implemented and tested in iteration 2.15. The original 47-method failure ledger +is fully resolved after iteration 2.36, but the complete feature is not yet +releasable because the wider conversion inventory remains open. +D030 records the original 58, D032 the 56-test baseline, D033 the 54-test +inference result and D034 the current diagnostic repair. +The 2.15 scoped gates passed 410 client tests plus two packaging +checks, 222 server tests and three seeded repeats of the eight new E2Es. +Ordinary Sender can recover extended backlog and enforce its required +capability. Remaining conversion and release-validation work stays explicit +in the design's implementation-status section. + +Iteration 2.16 validates file-backed process-restart recovery through ordinary +Sender without manually seeded frames. The final server gate passed 230 tests; +the two new recovery cases also passed three seeded repeats. Client production +code and the accepted 2.15 artifact are unchanged. D031 records scope and evidence. + +Iteration 2.17's first repair group is accepted: independent legacy conversion +comparisons are restored, with 251 scoped tests and three repeats of the 25 +affected tests passing. The broad 333-test selection still has 56 failures/errors; +functional fixes remain ahead. D032 separates fixture, product and environment +evidence and records the reviewed inference plan. + +Iteration 2.18 implements confirmed-missing inference for supported setters using +the existing binding and buffer. The final gates passed 414 client tests plus +two packaging checks, 261 server tests and three seeded repeats of 16 integration +and recovery tests. The original 333-test selection has 54 failures/errors, with +two removals and no new failing methods. D033 records scope and evidence. + +Iteration 2.19 repairs seven stale diagnostic expectations without production +changes. The 268-test server gate and three repeats of 23 integration/recovery +cases passed. The original 333-test selection has 47 failures/errors, with +exactly seven removals and no additions. D034 records independent legacy +rejection coverage, typed local recovery, evidence and remaining decisions. + +Iteration 2.20 characterizes LONG/UUID null-sentinel text behavior without +production changes. Three new tests and the 271-test server gate pass, as do +three repeats of 26 integration/recovery cases. The broad 47-method failure set +is unchanged. D035 corrects the predicted LONG text result and records a +proposed, not yet approved, source-null policy for these five conversions. + +D036 records the [remaining-failures plan](schema-aware-sender-remaining-failures-plan.md), +including all 47 baseline methods and the source-backed distinction between +nine test repairs and 38 missing-conversion cases. No further implementation +iteration or new test run is accepted by this planning entry. + +Iteration 2.21 accepts phase 1a: five expectation/fixture repairs and one new +public Sender floating-boundary test. The 277-test server gate and three +32-test repeats pass. The original 333-test diagnostic now has 42 failures, +with exactly the five planned removals and no additions. Production code and +the client artifact are unchanged. D037 records acceptance and remaining work. + +Iteration 2.22 accepts phase 1b: four direct-buffer fixtures now send explicit +legacy frames with exact wire and original stored-value assertions. The 281-test +gate and three 36-test repeats pass. The original 333-test diagnostic has 38 +failing methods, with exactly the four planned removals and no additions. +All nine phase-1 test repairs are complete; production code and the client +artifact remain unchanged. D038 records acceptance and the conversion backlog. + +## Working agreement — 2026-09-09 + +The primary agent owns architecture, scope, integration and acceptance. SOL +subagents receive bounded implementation, source-validation and independent +review assignments. Authors self-review; a separate reviewer and adversarial +reviewer challenge important changes before an iteration is accepted. Reviews +are evidence, not votes: findings must be resolved or explicitly justified. + +Each iteration follows this loop: + +1. State the smallest observable behavior being delivered and its non-goals. +2. Verify load-bearing assumptions in current source and existing tests. +3. Delegate implementation with explicit file ownership and acceptance tests. +4. Run the real client/server E2E path and relevant regression tests. +5. Review correctness, failure sequences, simplicity and developer experience. +6. Record findings, fixes, exact validation and what the iteration did not prove. + +Prefer existing components and direct control flow. Do not introduce a framework +for a single use case. Keep producer state producer-owned and replay immutable. +Review allocations, synchronization and repeated work on the steady-state write +path, but do not claim performance from inspection alone. Changes to a hot path +need a measured, reproducible comparison before performance claims are accepted. + +Tests use public API outcomes, protocol contracts, deterministic SQL results and +delivery behavior. No reflection, private-field inspection, test-only production +hooks, visibility changes solely for tests or implementation call-count checks. +Use bounded event-based coordination. A mocked peer is useful for wire faults, +not a replacement for real-server ingestion tests. + +No commits, pushes or external coordination are part of the current instruction. +Preserve pre-existing work; serialize builds sharing generated outputs. + +## Carried-forward product decisions + +- Early detection means converting supplied values against the server schema + before buffering, not merely comparing setter types or an acceptance mask. +- Keep QWP v1. New clients work with old servers in legacy mode and always + request schema support. Confirmation enables schema mode without an opt-out. +- Upgrade is one-way for a sender. Subsequent connections must support the + extension. No send-time transformation, downgrade encoder or SF rewriting. +- An in-progress row and completed blocks retain their encoding snapshot. + Adopt new schema or legacy-to-schema mode only at a real row boundary. +- A local conversion/lookup failure cancels the partial row, not completed + rows. A refreshed schema can require a whole-row retry; never splice a new + converter into an already partially converted row. +- Server DDL and auto-creation races remain server-authoritative. Metadata + feedback is not acceptance, permission to retry a terminal NACK or ACK progress. +- Schema mode uses target missing-value semantics; legacy mode preserves the + existing behavior. Conversion parity for supplied values needs shared vectors. +- Ingestion schema discovery follows write authorization, not SELECT permission. +- Schema caches grow on demand, with the agreed million-entry ceiling. No + subscriptions, polling or eager million-entry allocation. +- A real E2E test works from the first iteration. Negotiation, conversion + families, schema evolution and recovery are added and tested incrementally. + +## D001 — Start with the existing real-server test infrastructure + +Date: 2026-09-09. Status: accepted; iteration 1 completed and verified. + +Iteration 1 establishes a small public `Sender` → real QuestDB → SQL baseline +that subsequent schema work can extend. Use the sibling server's existing QWP +E2E harness, not a new embedded server framework in the standalone client. +Production wire and conversion behavior remain unchanged in this iteration. + +The baseline must exercise a precreated target schema and matching typed values. +Additional cases should establish meaningful row/flush and local-error recovery +contracts, without pretending that local STRING-to-UUID conversion exists yet. + +Reason: a green path through the actual client artifact, transport, WAL ingestion +and SQL is the earliest useful integration gate. It is not proof of the proposed +schema extension. Existing adjacent tests remain part of validation. + +Starting workspace: + +- Client: `/home/jara/devel/oss/java-questdb-client`, HEAD `981bdb02`. + Pre-existing untracked file: `design/schema-aware-sender.md`. +- Server: `/home/jara/devel/oss/questdb`, HEAD `12a33d651e`. + Pre-existing untracked directory: `core/rust/qdbr/parquet2/`; out of scope. +- Toolchain: Corretto JDK 25.0.4, Maven 3.9.11, Linux amd64. +- `/tmp` has limited free space. Use task-owned temporary test storage on the + workspace filesystem if needed; do not delete unrelated temporary files. + +Assignments: + +- `baseline_implementation` (SOL): implement the narrow real-server E2E baseline; + no production or build-configuration edits. +- `architecture_validation` (SOL): verify source assumptions and recommend the + smallest subsequent lookup/binding slice; read-only initially. +- `adversarial_review` (SOL): challenge failure sequences and compatibility + assumptions independently; read-only. +- Primary architect: journal, build/artifact provenance, scope and acceptance. + +## D002 — Test the actual standalone client artifact + +Date: 2026-09-09. Status: accepted and verified. + +The server also contains a separate `java-questdb-client` checkout. Do not assume +that source is the client under test. Build/install the standalone workspace's +`questdb-client:1.3.10-SNAPSHOT`, then run the server tests without activating its +`local-client` reactor profile. Check the packaged manifest, artifact hash and +Surefire classpath. No source copies or build-configuration edits are needed. + +Verified artifact SHA-256 (workspace JAR and installed JAR match): +`b9f547437a66825e11da10d31bd89b8e4092c49c30b4f2ed4b09bc9af58df969`. +Manifest commit: `981bdb02a471f3b290c89b8e78cbc422610e329e`. + +## D003 — Interpret review findings against iteration scope + +Date: 2026-09-09. Status: accepted. + +The adversarial review confirms that current code does not yet have first-write +mode selection, immutable schema generations or lookup gating. These are planned +production changes, not defects introduced by the baseline test. Do not inflate +the first iteration to implement all of them. + +Two findings do refine the next brief: + +- Minimal request/confirmation classification must accompany the first real + schema lookup. A lookup cannot be sent to an unconfirmed peer. Milestone 3 + adds the broader compatibility/transition matrix; it does not authorize an + unsafe handshake shortcut in milestone 2. +- A matching-type SQL success cannot demonstrate that schema lookup/binding + occurred. The first schema slice must also observe the wire contract and a + controlled lookup failure at the public setter, before any row is accepted. + +Do not add a generic rollback framework merely because a reviewer suggested +one. Establish the required A/invalid-B/C behavior across generations first; +use the smallest producer-owned state transition that satisfies it. + +## D004 — Make duplicate semantics explicit per mode + +Date: 2026-09-09. Status: source-verified; implementation remains future work. + +`QwpTableBuffer.getOrCreateColumn()` calls `lookupColumn(name, type)` before the +already-written check. `lookupColumn()` checks the bound type first. Thus current +legacy behavior ignores same-type duplicates but rejects a different setter +type, even when the column already has a value in that row. + +Schema mode's first-value-wins rule must check whether the target column was +already supplied before converting a second value, including another setter +type. Legacy mode must preserve its existing type-sensitive behavior. Add +black-box tests for both, including a valid UUID followed by an ignored malformed +UUID string in schema mode. Do not describe this as universally unchanged +duplicate behavior. + +SOL updated the design to revision 10 with this distinction and D003's minimal +confirmation prerequisite. These are source-backed clarifications of the agreed +modes, not a new opt-out, downgrade path or expansion of the first iteration. + +## Iteration 1 evidence and review + +Implemented by SOL: one method in the existing server `QwpSenderE2ETest`, +`testTypedUuidPreservesCompletedRowAfterLocalError`. The scenario is precreated +UUID target, completed A, partially written B with a local type error, explicit +flush, valid C omitting B's marker, flush/close, exact SQL containing only A/C. + +Executed from the standalone client workspace: + +```sh +mvn -B -pl core -Dtest=QwpTableBufferTest,QwpWebSocketEncoderTest,QwpWebSocketSenderTest -Dmaven.javadoc.skip=true install +``` + +Result: PASS, 151 focused unit tests and 2 `JarPackagingIT` tests, no failures, +errors or skips. The resulting snapshot is installed for the server E2E run. + +Executed from the server workspace: + +```sh +mvn -B -pl core '-Dtest=QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketTypeConversionE2ETest,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-tests.lOeKVW -Dmaven.compiler.showWarnings=false test +``` + +Initial result: FAIL, 59 tests, 1 setup error, no assertion failures or skips. +All 58 existing adjacent tests passed. The new test's +`connectWs(port, 0, 0, 0)` configuration was rejected with +`disabling auto-flush is not supported for WebSocket protocol`. + +Reflection: author and architect both assumed that the helper's all-off +translation was a supported configuration. The real artifact test disproved +that assumption before any schema code was added. Fix the test through supported +public configuration, keeping A demonstrably buffered at the local error. Do not +change production configuration rules or rely on timing to make the test pass. + +The corrected public configuration uses row threshold 2, disabled byte trigger +and the largest accepted interval (`Integer.MAX_VALUE - 1` milliseconds). Only A +completes before the first flush; B fails in its setter. Review the actual +interval rule, not the helper's apparent ability to disable every trigger. + +Second run of the same 59-test selection: 58 adjacent tests passed; the new +test reached SQL verification but failed `Expected no timestamp but found ts, +idx=2`. The stronger `.returns(...)` assertion also validates cursor metadata; +the query needs an explicit designated-timestamp expectation. Keep that stronger +assertion rather than reverting to `.returnsOnce(...)`. + +Independent and adversarial source reviews accept the A/B/C test shape and +confirmed meaningful leak detection and UUID limb-order coverage. They did not +catch the missing query metadata expectation. Source-review approval therefore +does not complete the acceptance gate; executed tests remain mandatory. + +Architect review adds one further discriminator: use public +`flushAndGetSequence()` results to prove that each explicit flush actually +publishes data. Final SQL after `close()` alone could conceal a no-op flush. +Assert a nonnegative first FSN and increasing second FSN, not an exact number +or internal call count. This also checks the claim that A was still buffered. + +Final focused command uses the same server build options above with +`-Dtest=QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError`. +Result: PASS, 1 test, no failures, errors or skips. The actual test was compiled +and executed against the freshly installed standalone client artifact. + +Both SOL reviewers accepted the final test independently after the FSN and +timestamp assertions were added. The final architecture check confirmed those +FSN comparisons are public publication semantics, not internal counters. The +adversarial check distinguishes publication (FSNs), delivery (close/ACK drain) +and stored state (SQL after WAL application). + +Final regression rerun used the same 59-test selection and temporary directory, +with `surefire:test` after the final focused `test` command had compiled the +current sources. Result: PASS, 59 tests, no failures, errors or skips: 2 sender +E2E tests, 1 receiver-side local-error test and all 56 existing type-conversion +E2E tests. Log: server `core/target/schema-aware-tests.lOeKVW/final-regression.log`. + +SOL then ran three serial repetitions of the final compiled baseline through +`surefire:test`, using explicit `(fuzz.s0, fuzz.s1)` seed pairs `(1, 2)`, +`(42, 43)` and `(1234567, 7654321)`. Each run exited 0 with 1 test and no failures, +errors or skips. The primary architect independently checked the logs, including +the recorded seeds and test summaries. These vary the existing harness's seeded +fragmentation; they are not a claim of exhaustive fuzz coverage. + +Repeat command, substituting each seed pair: + +```sh +mvn -B -pl core '-Dtest=QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError' -Dfuzz.s0=1 -Dfuzz.s1=2 -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-tests.lOeKVW surefire:test +``` + +Logs are retained in the server's task-owned +`core/target/schema-aware-tests.lOeKVW/`: `final-focused.log`, +`final-regression.log` and `repeat-1.log` through `repeat-3.log`. + +Acceptance: iteration 1 is complete. The final gates are 151 client unit tests, +2 packaged-JAR tests, 59 selected real-server tests and 3 extra seeded baseline +runs, plus independent and adversarial SOL review. Only the single server test +method and the client design/journal changed. No production or build +configuration changes, commits, performance claims or claims of completed schema +behavior. The unrelated Rust directory remains untouched. + +## Next iteration candidate + +Schema lookup plus matching-type binding on the same real E2E path. Exact scope +and prerequisites will be selected from source validation and adversarial +findings. Do not implement the entire conversion matrix or rewrite replay as +prerequisites for that first schema-aware write. + +Source-backed constraints for the next implementation brief: + +- Start with one precreated LONG target plus its timestamp handling. Include + minimal handshake confirmation and a classified lookup failure at a setter; + successful matching-type SQL alone is insufficient evidence of schema use. +- Reuse the existing cursor I/O loop for a bounded control-request exchange. + No second query client/socket or shared-socket lock. Controls must not enter + SF or consume data FSNs; classify schema replies before ordinary ACK parsing. +- Obtain WAL metadata through the sequencer-aware engine metadata API, not a + reader snapshot that may lag WAL DDL. Authorize ingestion-schema disclosure + with table-write authorization before emitting it. +- Keep immutable lookup metadata separate from producer-owned target bindings. + Bind/cache at row/layout boundaries; avoid per-value wrapper allocations or + a generic converter hierarchy for the first matching-type case. +- Assign and review bounded control-frame encodings before implementation. + Full table identities, ACK feedback and conversion families remain later + slices; their absence must be explicit, with no premature release/advertisement + of the complete extension. + +## D005 — Subdivide discovery from high-level Sender activation + +Date: 2026-09-09. Status: accepted; supersedes the immediate LONG-binding +candidate above, not the final design contract. + +Source review exposed a staging problem with the proposed LONG-only activation: +once the high-level Sender enters schema mode, other existing conversion paths +must either gain converters, fail explicitly or silently pass through. The last +choice violates the design; implementing every converter would make this a large +iteration. Merely turning on negotiation without binding is not schema mode. + +Iteration 2.1 therefore implements the negotiated discovery control plane and +production client codecs, exercised over the real low-level WebSocket client +against the real server. The ordinary Sender does not request this capability +yet and retains its existing behavior. There is no user disable switch, fake +server, build-time test switch or test-only production hook. The full extension +is still unreleased and must not be advertised as complete. Activation remains +a subsequent milestone-2 increment with explicit supported-conversion behavior. + +This is a smaller increment than the initial commentary proposed. It delivers +real discovery, authorization and wire/lifecycle tests while keeping the existing +public Sender E2E path green; it does not claim setter-time detection yet. + +Assignments: + +- `schema_server_slice` (SOL): server handshake, discovery, bounded response + lifecycle and dedicated real-server E2E tests. +- `schema_client_slice` (SOL): low-level handshake opt-in, production request/ + response codec and immutable metadata, with contract tests. No Sender or SF + changes and no unused producer/I/O mailbox ahead of its first caller. +- `architecture_validation` (SOL): independent adversarial review of protocol, + authorization, sequence/ACK isolation, bounds and response backpressure. +- Primary architect: wire decisions, scope, integration, source verification, + journal and final acceptance; second review of all final diffs. + +## D006 — Keep discovery separate from data sequencing + +Date: 2026-09-09. Status: frozen for iteration 2.1 implementation. + +Use the ordinary QWP1 twelve-byte header and version 1, with a reserved +`FLAG_CONTROL = 0x20`, zero tables and exact payload length. No other flag may +be combined with CONTROL. This flag is not the future persisted `FLAG_SCHEMA`. + +All multibyte numbers are little-endian. Request payload: + +```text +kind:u8 = 1 (DESCRIBE), requestId:i64 > 0, nameByteLen:u16, tableName:utf8 +``` + +Response payload: + +```text +kind:u8 = 2 (SCHEMA), requestId:i64 > 0, result:u8 +result: 0 KNOWN, 1 MISSING, 2 DENIED, 3 UNAVAILABLE, 4 TOO_LARGE +KNOWN only: + tableId:i32, metadataVersion:i64, designatedIndex:i16, columnCount:u16 + repeated columnCount times: + nameByteLen:u16, name:utf8, serverEncodedType:i32, + paramsLen:u16, params:bytes +``` + +The server type is the full encoded server ColumnType integer, not the QWP wire +type or a masked base tag. It preserves existing timestamp, decimal, geohash and +array parameters without a second lossy mapping. Freeze this representation as +part of the schema-control contract. The separate length-delimited extension +parameter area is empty in this increment; unknown extension bytes can be +retained/skipped by the client. Future converters interpret known target types; +the transport codec does not reject a schema merely for an unfamiliar type. + +Bounds: frames at most 1 MiB, at most 2,048 active columns, at most 1,024 extension +parameter bytes per column, names at most 381 UTF-8 bytes and 127 UTF-16 units. +Validate UTF-8 strictly and follow existing identifier rules. The initial agent +suggestion of a 127-byte limit was rejected: it would exclude valid multibyte +names. Reject truncation, trailing bytes, conflicting flags and overflow before +allocation. Oversized schemas return TOO_LARGE, not MISSING. + +Skip dropped columns and remap the designated timestamp index into the compact +active-column list; -1 means no designated timestamp. Return schema identity and +columns from one sequencer-aware metadata snapshot. Authorize insertion before +opening/serializing known-table metadata; write-only users need no SELECT grant. +Only an absent table token yields MISSING; metadata-open/drop races yield +UNAVAILABLE. Denials carry no schema. + +Recognize control traffic before allocating a data message sequence. It must +not pass through data processing, WAL commit, data ACK bookkeeping or SF. A +dedicated parked schema-response state owns bounded bytes until fully sent; +completion resumes ordinary pending ACK/durable-ACK processing. Tests must +interleave controls and data/deferred writes and exercise fragmented responses. + +Low-level client API uses one-way `requestQwpSchema()` and +`isQwpSchemaEnabled()`; there is no high-level Sender option. Request and confirm +headers follow the design. Missing confirmation is unsupported; malformed +confirmation is an error. Ordinary clients request nothing new in this increment. + +Refinements accepted during implementation review: + +- The 1 MiB ceiling includes the QWP header. Effective response size is also + bounded by the existing raw send buffer, including its WebSocket header. + Precompute using overflow-safe arithmetic; return TOO_LARGE before allocating + or serializing a schema that cannot fit. Reuse the current send machinery; + no additional growable reply queue or application-level chunking in 2.1. +- Unknown length-delimited extension bytes are skipped, not allocated/stored + per column. Full encoded ColumnType already preserves all current parameters. + Remove the speculative `getColumnParams()` API until a real consumer needs it. +- Validate request table-name syntax, but do not apply setter syntax bans to + returned column names: authorized existing SQL metadata must round-trip. +- Recognized malformed controls close with WebSocket protocol error 1002, not + a data NACK or UNAVAILABLE. Detect the control flag as soon as its byte is + available, including a truncated header. Live read-only metadata lookup yields + UNAVAILABLE without mutating role-change/data-delivery state. + +## Iteration 2.1 evidence and review + +The following records implementation and review chronologically. Intermediate +green runs are not acceptance; final acceptance and executed gates appear below. + +Pre-build review findings being resolved: + +- Main review: the schema-send catch also enclosed subsequent ACK sends. An ACK + blocked by backpressure could therefore have its saved ACK state overwritten + with RESUME_SCHEMA. Restrict the catch to the schema send and test interleaving. +- Main and independent review: substring-based HTTP header scanning could treat + a prefixed header name or another header's value as schema confirmation. Match + whole header names at line boundaries and reject duplicate confirmations. +- Independent review: malformed-field tests wrote at incorrect byte offsets, + sometimes rejecting for an unrelated reason. Correct offsets and retain + targeted tests that fail if the particular bound is removed. +- Main review: a blanket catch(Throwable) hid VM/programming failures as schema + unavailability. Map expected lookup failures explicitly instead. +- Independent review: include the new schema-send state in close-time draining, + and apply the configured output-buffer cap before full schema serialization. + +These are findings in unfinished drafts, not accepted behavior. They must be +resolved and validated before the increment is marked complete. + +Interim evidence (not acceptance): the focused standalone client install passed +189 unit tests and 2 packaged-JAR tests, with no failures, errors or skips. The +primary architect verified the Maven summaries and that workspace/installed JAR +SHA-256 values match: +`546873e4d03b9e98ba24aa3b9d3a25ed793369a8ab30d4aedb9861b1752feb61`. +Log: client `core/target/schema-client-2.1-rerun.RrLlpw/focused-install.log`. +The installed artifact contains the final `RESULT_DENIED` API and skips unknown +extension parameters. Manifest commit remains the working-tree base `981bdb02`; +the hash, not that base commit alone, identifies the changed artifact. + +Independent SOL source review accepts both production diffs after the response +state, close-draining, header parsing and exception-mapping corrections. The +architect's second review requested two additional codec tests: every truncated +prefix with a repaired envelope length, and the maximum multibyte response name. +Dropping a byte while retaining the old payload length only tests envelope +rejection; it does not demonstrate safety at each deeper field boundary. + +The first real discovery E2E run exposed invalid CREATE TABLE syntax in the +new test, not a production failure. After adding `PARTITION BY DAY`, the live +negotiated request/response passed. An extended rerun also passed immediate WAL +ALTER metadata freshness and full encoded DECIMAL type. Authorization, +malformed/unnegotiated traffic, deferred-commit isolation and forced response +backpressure remain mandatory before acceptance. + +Additional bounded-allocation review: do not size the temporary encoded-name +array from historical metadata column slots, which include tombstones. Count +active columns first, reject above 2,048, and allocate only for that bounded +active set. Compute the compact timestamp index during that count. This is a +resource bound, not a measured performance claim. + +The extra client codec tests passed: 191 focused unit tests, no failures, errors +or skips. Log verified by the architect: +`core/target/schema-client-review.oAb6mA/focused-test.log`. This was a test-only +rerun (`test`, not `install`); the installed production artifact is unchanged. +An agent initially reported a nonexistent log directory; the architect checked +the filesystem and corrected the evidence before recording it here. + +Interim server regression passed 96 tests, no failures, errors or skips, against +the installed standalone artifact. The architect checked the Surefire classpath +and per-class summaries in server +`core/target/schema-aware-discovery.compile/final-regression.log`. This includes +three discovery tests and 93 adjacent tests, but not the still-unwritten +authorization and interleaving cases. A filename containing `final` does not +make this the final acceptance run. + +Test-plan reflection: the initial interleaving proposal reused a large network +relay. The production low-level encoder and WebSocket client can express +DATA → DESCRIBE → commit directly, so a relay is unnecessary for that case. +Also, the existing configured send-fragmentation path deterministically parks +the response when it exceeds the chunk size, even with a writable kernel socket. +Use those existing seams for schema-response resume. Testing an ACK blocked +immediately after a fully sent schema response additionally requires a small +schema response and deterministically coalesced input; merely sending two frames +quickly is not evidence that the server processed them in one read. + +Authorization E2E passed two tests over the real client/server connection: +write-only discovery succeeds despite SELECT denial, and INSERT denial returns +DENIED with no schema identity/columns. Log checked by the architect: server +`core/target/schema-acl.z7FCbt/acl-e2e.log`. The fixture uses the existing +SecurityContextFactory boundary; no production hook or visibility change. +Review tightened the SELECT-denial precondition to an authorization exception, +and ensured helper-created clients close if their handshake fails. + +Review of the first state-test draft found two false-positive risks despite its +green run. A schema-response barrier proves the server handled the preceding +frame, but does not prove a wrongly committed WAL transaction was applied; +drain WAL before asserting zero visible rows. Also, chunk-size 1 proves schema +send resumption but cannot exercise an ACK blocking inside the initial schema +handler: the schema itself always parks first. Keep separate tests for these +distinct response states and require the observed ordering appropriate to each. + +The complete new schema matrix subsequently passed 11 tests: six discovery/ +protocol/deferred/backpressure cases, two authorization cases and three bounds/ +read-only cases. Log verified by the architect: server +`core/target/schema-aware-discovery.compile/full-schema-matrix-final.log`. +Public SQL creation of 2,049 columns succeeds in this checkout, so the active +column-limit test reaches discovery's TOO_LARGE result without internal metadata +fabrication. A 512-byte send buffer rejects a wider response while allowing a +subsequent small lookup on the same connection. A controlled live read-only +engine boundary yields UNAVAILABLE, then KNOWN after re-promotion; this is not +an exhaustive endpoint-failover test. + +The initial coalescing socket fixture failed test compilation because its +constructor exposed SLF4J through a module the server tests do not read. The +final response-order fixture uses a small raw loopback WebSocket peer with +production QWP encoding/decoding. No build configuration was changed. An early +two-write fixture also allowed the ACK to arrive before the control; the final +fixture writes the combined pipeline and explicitly asserts SCHEMA → ACK → +next SCHEMA. One TCP write is not a universal one-read transport guarantee; +the observed response order is the test's discriminator. + +## Iteration 2.1 acceptance — 2026-09-09 + +Accepted after SOL implementation, independent/adversarial source review, +cross-review of the final test fixtures, and the architect's source/evidence +checks. No outstanding blocker remains within this increment's scope. + +Client commands (standalone client workspace): + +```sh +mvn -B -pl core -Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest -Djava.io.tmpdir=core/target/schema-client-2.1-rerun.RrLlpw -Dmaven.javadoc.skip=true install +mvn -B -pl core -Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest -Dmaven.javadoc.skip=true -Djava.io.tmpdir=core/target/schema-client-review.oAb6mA test +``` + +The install passed 189 unit tests plus 2 packaging tests; the final test-only +additions raised the focused unit selection to 191, all passing. Logs and the +unchanged installed artifact hash are recorded above. JDK Unsafe/removal and +native-access warnings were present; no build failure was hidden by them. + +Fresh final server regression, after restoring the correct production source: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-discovery.compile test +``` + +Result: PASS, 104 tests, no failures, errors or skips. This used the full `test` +lifecycle to compile current main/test sources, without the `local-client` +profile. The architect checked the standalone installed-JAR classpath and log +`core/target/schema-aware-discovery.compile/final-regression-104.log`. + +Three additional serial repetitions used the freshly compiled sources: + +```sh +mvn -B -pl core -Dtest=QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest -Dfuzz.s0=1 -Dfuzz.s1=2 -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-aware-discovery.compile surefire:test +``` + +Seed pairs `(1,2)`, `(42,43)` and `(1234567,7654321)` each passed all 11 tests, +with no failures, errors or skips. Logs: `schema-repeat-1-2.log`, +`schema-repeat-42-43.log`, `schema-repeat-1234567-7654321.log` in the same server +directory. These are stability checks, not exhaustive fragmentation coverage. + +Adversarial mutation check: temporarily move the pending ACK pumps back into +the schema-send catch. The completed-schema/chunk-24 test then fails as intended: +the next DESCRIBE receives a duplicate 11-byte ACK rather than SCHEMA. The +observed exception is `Malformed QWP schema response: invalid frame length: 11`. +Log: `mutation-ack-catch.log` in the same directory. Restore with the exact inverse +patch before the final regression. The restored upgrade processor's SHA-256 +matches its pre-mutation value, independently checked by the architect: +`ec0e15bc6c8cc79be96f4a39152ad302596772f6a8f0996daf4ddd9147e5c9c0`. +This confirms the test detects the reviewed state-loss bug, not just happy-path +response delivery. No deliberate mutation remains in the workspace. + +Scope at handoff: discovery control frames and low-level client negotiation/ +codecs are complete. Normal Sender activation, local target conversion, schema +bindings, ACK/NACK schema feedback, persisted framing and replay changes remain +future work. No send-time transformation, opt-out, subscription/cache framework +or public feature-completion claim was introduced. No performance improvement +is claimed. No commits or pushes were made; unrelated Rust files and the accepted +iteration-1 baseline were preserved. Both repositories pass `git diff --check`. + +## Next increment + +Integrate lookup with producer-owned immutable schema bindings and a first real +setter-time failure E2E. Select the activation boundary and supported conversion +slice from source before changing the high-level Sender. Do not silently pass +through conversions merely to enable negotiation, and do not expand this into a +send-time transformation or replay rewrite. + +## D007 — Test the conversion component before Sender activation + +Date: 2026-09-09. Status: accepted and verified in iteration 2.2. + +Do not activate normal Sender merely to exercise a converter. Its flush path +encodes and durably appends bytes before I/O sends them. Activating schema mode +therefore also needs coherent first-use negotiation, row boundaries, frame +identities, reconnect and persisted-backlog compatibility. Adding FLAG_SCHEMA +but deferring its recovery checks would violate the design's first-framing-change +gate. Implementing all of that alongside the first converter is too large a +single increment. + +Instead add the genuine production `QwpSchemaRowBuffer` component, holding one +immutable server snapshot and a private sparse existing `QwpTableBuffer`. Its +first supported pairs are STRING-to-UUID, typed UUID-to-UUID and matching LONG. +Unknown targets, missing columns and unsupported non-null pairs fail explicitly, +not through raw-value fallback. Null string input is an effective target NULL +for the supported targets, matching current QWP setter semantics. `atNow()` +retains server-assigned time; explicit timestamp conversion is not in this slice. + +The E2E path is real DESCRIBE → immutable binding → locally converted A / invalid +B / valid C → existing typed UUID/LONG QWP encoding → real server ACK and SQL. +Observe typed UUID bytes, not merely successful server-side parsing. Failures +cancel B and its newly introduced buffer columns, not completed rows. Duplicate +suppression uses the target column before validating/converting a second input, +including cross-setter duplicates. Keep the mutable storage private and expose +only row operations and encoding; no per-value wrapper allocation or new queue. + +This is deliberately conversion-component E2E, not full schema mode. It does +not transmit schema identities, activate Sender, add a lookup mailbox/cache or +change SF. The existing type definitions faithfully describe the already +converted bytes; no STRING value is passed through as a substitute for UUID +conversion. Design revision 12 makes this staging distinction explicit. + +Reflection: initial agent briefs coupled any schema-bound component to the new +persisted flag. The architect challenged that coupling: wire identities enable +feedback and snapshot accounting, but are not required to test a standalone +local converter honestly. Independent review agreed after separating component +behavior from activation claims. Another brief proposed rejecting a stale +identity; this was rejected outright. The design says identity is descriptive, +not CAS, and a version mismatch alone must not reject otherwise valid data. + +Assignments: + +- `schema_client_slice` (SOL): row component, classified schema exceptions, + minimal UUID conversion helper if needed, codec refinement and component tests. +- `schema_server_slice` (SOL): real-server component/UUID conformance E2E tests; + no server production or framing changes. +- `architecture_validation` (SOL): independent/adversarial API, ownership, + rollback, unknown-parameter and test-discrimination review; read-only. +- Primary architect: scope, source verification, design/journal, integration, + artifact provenance and final acceptance. Build slots remain serialized. + +## D008 — Preserve unknown extension presence for the first real consumer + +Date: 2026-09-09. Status: accepted for iteration 2.2. + +Iteration 2.1 correctly skipped opaque per-column extension bytes without +allocating/storing them. A converter now needs one additional fact: whether +such bytes were present. Otherwise an unfamiliar extension on a familiar base +type could be silently ignored when writing that column. + +Retain a boolean per column and expose +`hasColumnExtensionParameters(index)`. Continue skipping the bytes and keep the +snapshot immutable. Reject use of a parameter-bearing column with +UNSUPPORTED_FEATURE; allow unrelated supported columns. Known parameters remain +packed in the full ColumnType integer. The first converter compares exact +supported types, not masked tags that could discard unfamiliar flags. This is +a consumer-driven refinement, not restoration of speculative byte arrays/API. + +## D009 — Compare UUID conversion with QWP ingestion, not SQL casts + +Date: 2026-09-09. Status: source-verified and runtime parity verified in iteration 2.2. + +Server `WalColumnarRowAppender.putStringToUuidColumn()` checks the null bitmap, +then calls strict UUID length/dash/hex parsing. An empty non-null string fails. +`SqlUtil.implicitCastStrAsUuid()` instead treats an empty string as null, so it +is not the correct conformance oracle. Java's UUID parser is not substituted. + +Reuse the narrow server parsing rules with the client's existing hexadecimal +parser. Parse both limbs before append. The public input order is `(lo, hi)`; +existing `ColumnBuffer.addUuid(hi, lo)` writes low then high. Preserve the native +sentinel limitation: both UUID limbs equal to Long.MIN_VALUE represent NULL; +a single such limb does not. Matching LONG likewise preserves its null sentinel. + +Use one versioned UUID vector corpus, vendored byte-identically into both test +resource trees, so each repository's tests are hermetic. Verify equal hashes. +Compare acceptance/rejection and stored values against actual legacy QWP string +ingestion as well as the new local component. Invalid input must never become +a successful null merely because a different SQL conversion would accept it. + +## Iteration 2.2 evidence and review + +Development evidence follows; final acceptance is recorded below. Do not infer +high-level Sender activation from component completion. + +Read-only baseline, against the previously installed 2.1 client and compiled +server sources: + +```sh +mvn -B -pl core '-Dtest=QwpWebSocketTypeConversionE2ETest#testStringToUuidColumn+testUuidToStringColumn+testUuidToVarcharColumn' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-binding-baseline.bAkuPn surefire:test +``` + +PASS: three tests, no failures, errors or skips. Log checked by the architect: +server `core/target/schema-binding-baseline.bAkuPn/uuid-baseline.log`. + +Draft review caught input-pair validation running before duplicate suppression. +Fix by resolving the actual target wire type, asking the existing sparse buffer +for that target column, and honoring its duplicate result before validating the +input pair. Do not introduce a parallel row-stamp array or another row tracker; +one existing bookkeeping source suffices. Other draft checks cover closed-state +reuse, table/column validation, ambiguous schema names and failure cleanup. + +Keep the design's five exception reasons. In this component, confirmed MISSING +means inference is not implemented, and TOO_LARGE means discovery cannot supply +the bounded schema; both are UNSUPPORTED_FEATURE with distinct diagnostics. +DENIED is ACCESS_DENIED and transient UNAVAILABLE is SCHEMA_UNAVAILABLE. +Do not invent staging-only public reason codes or call these INVALID_VALUE. +Reserved SCHEMA_CHANGED is not emitted until actual refresh/rebinding exists. + +Independent review also caught a test-client leak if the handshake failed before +the helper returned, and a conformance gap: only invalid strings initially went +through the legacy server converter. The helper now closes on construction +failure. The E2E test compares every accepted value, explicit null and native +sentinel through both local binary UUID encoding and actual server VARCHAR-to- +UUID ingestion; each invalid vector gets a fresh stream and must receive a NACK. + +First client build passed 200 unit tests and two packaging tests. The architect +checked the log and matching workspace/installed JAR hash. Before accepting the +slice, test review requested stronger wire assertions: an encoder returning a +positive length alone does not prove row retention, null encoding or rollback. +The server test must assert exactly the A/C rows and their effective columns, +so a failed-row-only column cannot remain unnoticed as an all-null definition. +Client tests must likewise inspect observable bytes after partial-row rejection, +cancel/reset and explicit nulls. Final evidence follows after these checks run. + +Server validation caught two test authoring mistakes before acceptance. The +first compile lacked required `WebSocketFrameHandler.onClose()` callbacks. The +first executed run then passed two of four tests; the other two expected `NaN` +for LONG nulls, while this server's SQL renderer returns `null`. The architect +checked both actual/expected outputs: the UUIDs, retained A/C rows and values +matched, and only the null text differed. Fix the expected text, not production +conversion semantics, and rerun the tests and regression selection. + +The client build also caught a server/client API difference (`NumericException` +uses `instance()` in this client, not `INSTANCE`) and a malformed test resource +with literal `\\t` text instead of actual tab separators. Both were corrected +before the successful install. Failure logs are retained as +`core/target/schema-row-buffer.2.2/focused-install.log` and +`focused-install-rerun.log`; the architect checked the actual failures. + +## Iteration 2.2 acceptance — 2026-09-09 + +Accepted after author self-review, independent SOL adversarial/source review, +architect source review and direct checks of build logs, artifact provenance +and corpus hashes. No concrete blocker remains within the component scope. + +Client install (200 unit tests plus two packaging tests, all passing): + +```sh +mvn -B -pl core -Dtest=QwpSchemaRowBufferTest,QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest -Djava.io.tmpdir=core/target/schema-row-buffer.2.2 -Dmaven.javadoc.skip=true install +``` + +Log: `core/target/schema-row-buffer.2.2/focused-install-final.log`. +After test-only strengthening, the same selection passed **203 tests** on final +sources with `test` instead of `install` and temporary directory +`core/target/schema-row-buffer-final.2.2`. Log: +`core/target/schema-row-buffer-final.2.2/final-focused-test.log`. +The intermediate component/codec-only run passed 21 tests; its log is +`core/target/schema-row-buffer-tests.2.2/strengthened-test.log`. +All green runs have zero failures, errors and skips. Production did not change +after installation; the final test-only changes did not require reinstallation. + +Workspace and installed `questdb-client-1.3.10-SNAPSHOT.jar` SHA-256: +`eeec24a05e186cb22c9b063008a5febf1bf5b57bd640d8fe9f24d82669d26251`. +The manifest records base commit `981bdb02a471f3b290c89b8e78cbc422610e329e`; +the hash, not that base commit alone, identifies this dirty-worktree artifact. +Server Surefire reports explicitly reference the installed standalone artifact +under `/home/jara/.m2/repository/org/questdb/questdb-client/1.3.10-SNAPSHOT/`. +No embedded-client profile was used. + +Fresh server regression, against server base +`12a33d651e51e2682e7a448c8db5168fc72dfad3` plus the task changes: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaRowBufferE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-row-buffer-server test +``` + +PASS: **108 tests**, zero failures, errors or skips. Log: +server `core/target/schema-row-buffer-server/final-regression.log`. +The four new E2E tests also passed separately in `new-e2e-final.log`. +Three additional serial runs used `-Dtest=QwpSchemaRowBufferE2ETest`, +`-Dfuzz.s0= -Dfuzz.s1=` and `surefire:test` against those compiled +sources. Seed pairs `(1,2)`, `(42,43)` and `(1234567,7654321)` each passed all +four tests. Logs: `repeat-1-2.log`, `repeat-42-43.log` and +`repeat-1234567-7654321.log` in the same server directory. These are stability +checks, not an exhaustive fuzzing or performance claim. + +The shared 20-case corpus is at test resource +`io/questdb/client/cutlass/qwp/uuid-string-conformance.tsv` in each repository. +Both copies have SHA-256 +`c0ce3c5836cc58e1960dbcadbcfd9d53df0d842af796c76d25d942259cec90e4`. +Tests inspect public protocol bytes and deterministic SQL, without reflection, +private-state hooks or runtime cross-repository resource dependencies. + +Delivered scope: immutable schema-bound row construction, STRING-to-UUID, +typed UUID/LONG writes, explicit target nulls, duplicate-first semantics, +partial-row rollback and stable classified exceptions. Opaque extension presence +is retained without storing its bytes; unknown used types are rejected locally. +The mutable row storage remains private and reuses existing row bookkeeping. + +Remaining scope: ordinary Sender activation and lookup coordination, inferred +schemas, further conversion families, schema identities in persisted frames, +ACK/NACK schema feedback, snapshot transitions and one-way-upgrade/recovery +integration. Caller-owned response correlation and snapshot acquisition remain +explicit component preconditions. No send-time transformation, feature opt-out, +subscription machinery or parallel row-state tracker was introduced. +No existing Sender, I/O, encoder, SF or server production behavior changed in +iteration 2.2. Both worktrees pass `git diff --check`; unrelated work remains +preserved. No commits or pushes were made. + +Next architectural gate: define the smallest pinned-frame/recovery increment +before activating Sender. Introducing persisted schema framing must include +negotiation and replay/recovery checks in the same increment. Continue to keep +the real component and public-Sender baseline E2E paths green throughout. + +## D010 — Persisted schema framing and replay before Sender activation + +Date: 2026-09-09. Status: accepted in iteration 2.3; evidence below. + +After the user asked what unchanged ordinary Sender behavior meant, the +architect checked whether this increment could safely activate it. Source +review and independent SOL review found that UUID/LONG-only activation would +either reject the other existing conversions globally on supporting servers or +silently pass values through. The latter violates the design; the former is +not an acceptable ordinary-Sender regression. Do not use a per-table, SF or +configuration exception as an implicit feature opt-out. + +Complete the next prerequisite instead: extended table framing and safe SF +recovery. This is still not high-level schema activation, and is not described +as the final prerequisite: lookup coordination, conversion coverage, snapshot +transitions and feedback remain. The public Sender baseline stays green. + +Wire decision, captured in design revision 13: `FLAG_SCHEMA=0x02`; after each +table name, kind 0 means unknown, or kind 1 precedes a non-negative i32 table +ID and i64 metadata version, little-endian. Reject invalid kinds/fields, +SCHEMA+CONTROL and schema-flagged table-less frames. Reject unnegotiated schema +frames before data processing. Stale identities remain descriptive, not CAS. +ACK metadata feedback is not needed to parse/persist these identities and is +not implemented in this slice. + +The encoder gets explicit per-operation schema methods. Do not turn the +existing mutable flags setting into a sticky schema mode that can relabel +later legacy data. A message has one layout; reject mixing legacy/schema table +addition methods. Split messages retain the original flag and table bytes. +The schema row component passes its immutable snapshot identity to the encoder; +there is no new mutable identity field on QwpTableBuffer merely for serialization. + +Recovery derives its requirement in the existing ordered frame scan, before +any reconnect can send dictionary catch-up or a legacy prefix. Any unacknowledged +retained extended frame requires support, conservatively including a deferred +tail; acknowledged residue does not. Foreground and background/drainer paths +must enforce the requirement for their own slot on every connection. A mismatch +retains bytes and watermarks and must not become quarantine, stripping or +downgrade. Fresh ordinary Sender still does not request the extension in this +staging increment; schema-required replay does request it. No second SF scan or +new persistent side file is planned. + +Assignments: SOL client author owns explicit encoding and SF/reconnect safety; +SOL server author owns bounded parsing/ingress and real-server tests; independent +SOL reviewer challenges lifecycle, compatibility and test discrimination. +The architect owns the contract, journal, source checks and final acceptance. +Tests must cover actual disk recovery and byte-identical replay, not just flag +getters, and preserve the existing component and public-Sender E2E coverage. + +## Iteration 2.3 development evidence + +Pre-edit compiled baseline, run without recompiling while authors prepared the +new source changes: + +```sh +mvn -B -pl core '-Dtest=RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-framing-baseline.LGBRLN surefire:test +``` + +PASS: 27 tests, zero failures/errors/skips. Log: +`core/target/schema-framing-baseline.LGBRLN/baseline.log`. This establishes the +unchanged recovery baseline; it is not validation of the new source edits. + +Source review found local deferred-tail retirement in both BackgroundDrainer +and the cursor loop before any connection. The conservative schema-required +tail rule must gate that self-acknowledgment too, otherwise an incompatible +endpoint can coincide with watermark advancement and deletion without sending +any bytes. Keep legacy-tail retirement unchanged. A supporting connection may +still abort the orphaned transaction normally; tests must not expect deferred +rows without a commit to be ingested. + +Draft encoder review found transient mutation of configured flags around header +writing. A header allocation failure could leave FLAG_SCHEMA set for a later +legacy encode. Pass computed header flags explicitly instead, including the +existing delta-dictionary path. Reject schema/table-add layout mixing and +schema-flagged table-less construction; do not add another buffering framework. + +Independent and architect review caught schema-capability errors being routed +through the existing durable-ACK mismatch budget. That budget can quarantine an +orphan and report data loss, which is wrong for schema-required bytes. Schema +mismatch must retain/retry until stop or a compatible endpoint, and reset—not +consume—the unrelated durable-capability episode. Preserve typed errors through +endpoint exhaustion so the correct policy sees them. + +One-way support is owner-wide, not just a flag on an individual reconnect +factory. A new legacy-only orphan connection cannot go to an old server after +the owning sender required/confirmed schema support; the reverse ordering also +matters. Keep the recovered per-slot requirement and one shared monotonic sender +requirement. Existing legacy connections need not be forcibly closed solely for +another connection's upgrade, but every subsequent connection must respect the +shared requirement. Tests need both foreground/background orderings. + +The architect and independent reviewer also found a publication race: checking +the requirement only at the start of trySendOne allows the producer to publish +the first extended frame after that check. Recheck after observing the complete +published frame, before delta processing or send. The producer's requirement +write precedes ring publication, so this ordering makes the guard effective. +Keep the earlier guard for retirement/catch-up paths. Do not add private test +barriers to force this exact instruction interleaving; combine public live- +append coverage with the explicit memory-ordering argument. + +Structural cost budget: a known identity adds 13 bytes per table block, unknown +adds one byte, never per row. Recovery folds one boolean into its existing +ordered scan; it does not reread the SF log. Capability checks are per frame or +connection, not per value. These are source/protocol observations, not measured +throughput, latency or allocation-improvement claims. + +A custom reconnect factory may ignore the new default requireSchema callback. +The background drainer must check the returned connection inside its typed +retry loop, using the actual recovered engine requirement. Checking only at +send-loop startup would let a capability mismatch escape through generic setup +failure handling and mark the slot failed. Independent review accepted the +real-network regression: a factory returns old-server connections twice, stop +terminates retries, no frames or data-loss errors occur, and reopening the disk +slot proves both watermarks and schema-required backlog were retained. + +First integrated client install passed 294 unit tests and two packaging checks, +with zero failures, errors or skips. Log: +`core/target/schema-framing-final.2.3/focused-install.log`. The architect checked +the aggregate results and matching workspace/installed main-JAR SHA-256: +`6e1d9067594db8afd70c96d0cbd7b999eed56a18c1a16663c686d6bce1832340`. +This artifact unblocked fresh server compilation; it is not final acceptance. +Explicit owner-wide negotiation and split/deferred identity-preservation tests +are still required. The real-network tests already cover incompatible peers +that also lack durable ACK, exact mixed legacy/schema replay after recovery, +and deferred-tail retention followed by normal local abort on a supporting +connection. No new white-box production hooks were introduced. + +Fresh server validation against that standalone artifact passed 265 tests: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaIdentityParserTest,QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide,QwpUdpMalformedTest,QwpUdpInsertTest' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-identity-server test +``` + +Log: server `core/target/schema-identity-server/final-regression.log`. +The architect verified zero failures, errors and skips, and the standalone +`/home/jara/.m2/repository/org/questdb/questdb-client/1.3.10-SNAPSHOT/questdb-client-1.3.10-SNAPSHOT.jar` +in the identity E2E Surefire runtime classpath. The earlier fresh focused run +passed 61 tests; log `core/target/schema-identity-server/focused.log`. +Three further serial runs used `-Dtest=QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest`, +`-Dfuzz.s0= -Dfuzz.s1=` and `surefire:test` against those compiled +sources. Seeds `(1,2)`, `(42,43)` and `(1234567,7654321)` each passed nine tests. +Logs: `core/target/schema-identity-server/repeats/repeat--.log`. +These are stability checks, not exhaustive fuzzing. UDP regression is included +because the shared parser now understands schema framing; both ordinary and +Linux batched UDP receivers must reject it without WebSocket negotiation. + +Test review rejected version-zero/table-less pseudo-frames as positive schema +recovery fixtures and a fixed 100 ms checkpoint sleep. Use real encoder output +for both schema frames and the retained legacy suffix, with lengths matching +the allocations. Observe the public ACK watermark with a bounded wait before +reopening. Deliberately malformed delta-parser fixtures remain synthetic. +These corrections strengthen the evidence; a green run of the earlier fixtures +alone does not establish valid-frame recovery correctness. + +## Iteration 2.3 acceptance + +Date: 2026-09-09. Accepted by the architect after SOL author self-review, +independent source/adversarial review, and final-source runtime verification. +No remaining finding blocks the frozen scope. Both worktrees remain uncommitted; +unrelated server `core/rust/qdbr/parquet2/` was not touched. + +Final client command: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-framing-acceptance.2.3 -Dmaven.javadoc.skip=true test +``` + +PASS: **298 tests**, zero failures, errors or skips. Log: +`core/target/schema-framing-acceptance.2.3/focused-test.log`. +The preceding final-source run passed 297 before the live-append test was added; +log `core/target/schema-framing-final-source.2.3/focused-test.log`. +The two packaging checks passed during the earlier install recorded above. +Production did not change after that install; final edits were test-only. +The architect rechecked the matching workspace and installed main-JAR SHA-256: +`6e1d9067594db8afd70c96d0cbd7b999eed56a18c1a16663c686d6bce1832340`. +The manifest still records base commit +`981bdb02a471f3b290c89b8e78cbc422610e329e`; the hash identifies the dirty-worktree +artifact. Server validation remains the fresh **265-test** gate recorded above, +with three additional nine-test seeded E2E passes against this same artifact. + +The final five-test client network suite passed three more serial runs using +`-Dtest=QwpSchemaReplayNetworkTest`, `surefire:test` and +`-Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-framing-repeats.2.3/run-`. +Logs: `core/target/schema-framing-repeats.2.3/run-1.log`, `run-2.log`, `run-3.log`. +No seed or forced-interleaving claim applies to these repetitions. + +The last live-append test starts the actual cursor loop on an already-connected +legacy peer with an empty disk engine, then publishes the first valid extended +frame through appendBlocking. It observes repeated connection attempts, no +binary frames at the peer, no terminal/quarantine outcome, unchanged ACK and +publication watermarks, and retained schema-required backlog after reopening. +Together with the post-publication guard's memory-ordering argument, this covers +the important transition without a private barrier or new production test hook. +The acknowledged-residue fixture also explicitly proves FSN 0 remains represented +in the recovered segment before checking that it does not require schema mode. + +Delivered: immutable encoding identity survives table framing, splitting and +disk replay; negotiated server parsing accepts descriptive stale identities; +unnegotiated WS and all UDP ingress reject extended framing; foreground and +background reconnects share a one-way requirement; capability mismatch retains +the backlog rather than transforming, quarantining or acknowledging it away. +Existing legacy wire bytes and durable-ACK failure policy retain regression +coverage. No protocol-version bump, send-time transformation, feature switch, +second recovery scan or persistent capability side file was introduced. + +Reflection: persisted-format changes reach beyond parsing. Startup, orphan-tail +self-acknowledgment, mixed backlog prefixes, shared connection ownership and +producer/I/O publication all need the same safety contract. The small explicit +encoder API and existing recovery scan were sufficient; a parallel buffering or +recovery framework was unnecessary. Source review found the lifecycle mistakes +before acceptance; valid-frame and real-network tests now guard their outcomes. +No throughput or latency improvement was measured or claimed. + +Remaining product work is explicit: ordinary-setter activation, producer-owned +lookup coordination, inferred schemas, the remaining conversion families, +snapshot transitions, and ACK/NACK schema feedback. Next, inventory conversion +coverage against server source and choose a bounded family with real-server +conformance tests. Do not activate the UUID/LONG-only component globally or +introduce an implicit opt-out/raw fallback to make incomplete activation pass. + +## D011 — LONG-input numeric conversion family + +Date: 2026-09-09. Status: accepted in iteration 2.4 on 2026-09-10, with the +sentinel parity assumption corrected by D012 and the approved rule in D013. + +Three SOL discovery briefs proposed different slices: STRING/LONG to INT/LONG, +STRING/LONG to all integer widths, and LONG to all six scalar numeric targets. +The architect selected the last: extend the existing schema row component's +longColumn to BYTE, SHORT, INT, LONG, FLOAT and DOUBLE. This follows one coherent +server dispatch without introducing distinct string grammars or a private SHORT +parser in the same increment. The narrower text proposals remain useful future +work, not rejected product functionality. + +Source authority: server `cairo/wal/WalColumnarRowAppender.putIntegerToNumericColumn` +and `cutlass/qwp/protocol/QwpFixedWidthColumnCursor.isCurrentValueSentinelNull`. +The architect read both after the independent reviewer identified the shared +dispatch. Source LONG_NULL is classified before range checks or casts. Correct +the initial client brief's suggestion to treat every raw LONG as an ordinary +number: Long.MIN_VALUE must become target missing/null, not overflow or a large +negative FLOAT/DOUBLE. BYTE/SHORT store zero for null; INT/LONG use their null +sentinels and floating targets use NaN. A non-null LONG equal to Integer.MIN_VALUE +is within range and becomes the INT null sentinel after narrowing. Rejecting +that value would diverge from current QWP behavior. + +BYTE, SHORT and INT use inclusive Java signed ranges. LONG is identity. +FLOAT/DOUBLE use the same Java casts as ingestion; precision loss is accepted, +including 16,777,217 to float 16,777,216 and 9,007,199,254,740,993 to double +9,007,199,254,740,992. Encode the target wire type directly. No fallback that +leaves the server to perform the new conversion is permitted. + +Keep target-to-wire resolution small and shared so an already-written column +ignores duplicate setters before pair/range/value validation, including a +different existing setter on a newly supported target. The existing explicit +null behavior of stringColumn(null) may cover the new supported targets; +non-null numeric string parsing remains UNSUPPORTED_FEATURE unless the write +is an ignored duplicate. Used unknown/parameterized targets remain explicit +limitations, not INVALID_VALUE. Range failures cancel the entire partial row +and its new-only bindings, preserving completed rows through the existing +rollback machinery. No extra row-state tracker or converter framework. + +Assignments: SOL client author owns QwpSchemaRowBuffer, unit tests and canonical +shared vectors; SOL server author owns real-server tests and an identical +vendored resource, with no server production changes; independent SOL reviewer +challenges parity, sentinels, duplicate ordering, rollback and evidence quality. +Client and server Maven runs remain serialized within their repositories. Server +tests use the freshly installed standalone client, never the embedded-client +profile. The architect owns this journal, scope and final acceptance. + +Acceptance requires explicit expected wire values/types/null bits and SQL +results, not only equality between two paths. Exercise accepted and rejected +vectors through both local target-typed conversion and legacy raw LONG server +conversion; rejected server batches use an isolated connection. Include integer +boundaries and one-outside values, floating precision boundaries, source nulls, +target sentinel collisions, omissions, duplicate-before-overflow and A/invalid-B/C +rollback with B-only columns absent. Keep the accepted framing/recovery and +ordinary Sender E2E regressions green. Tests use public component/wire/SQL +contracts, no reflection, production test hooks or arbitrary sleeps. + +This is new functional coverage, not a performance optimization or bottleneck +claim. Check successful conversion paths for per-value object allocation and +unnecessary scans; do not claim measured throughput/latency improvements. +Deferred: non-null text parsing, floating-source conversions, timestamps, +decimals, arrays, all other families, lookup coordination and ordinary Sender +activation. Design revision 14 only clarifies the historical staging paragraph: +component E2E need not initially transmit identities, but accepted iteration 2.3 +now does. The component's stale framing Javadoc will be corrected with its edit. + +## Iteration 2.4 development evidence + +Pre-edit component baseline: + +```sh +mvn -B -pl core -Dtest=QwpSchemaRowBufferTest -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-baseline.2.4 -Dmaven.javadoc.skip=true test +``` + +PASS: 12 tests, zero failures/errors/skips. Log: +`core/target/schema-numeric-baseline.2.4/baseline.log`. Maven ran the normal +lifecycle but found compiled classes up to date; this checks the accepted +pre-edit component, not the new conversion implementation. + +The architect found a null-ordering bug in the first draft. Generalized wire-type +resolution also accepts UUID for the existing UUID setter; an unconditional +LONG_NULL shortcut therefore accidentally accepted LONG-to-UUID as UUID null. +Validate that the source/target pair is supported after duplicate suppression +but before interpreting the source null sentinel. Unsupported pairs remain +unsupported even for LONG_NULL. A duplicate LONG_NULL after a valid UUID value +must still be ignored. Dedicated regressions cover both outcomes and rollback. + +This differs from the intentional stringColumn(null) behavior frozen in D011: +that existing explicit-null operation is allowed on all supported numeric +targets. Non-null numeric strings are still unimplemented. Independent review +challenged the scope; the architect retained the documented target-owned null +rule and required null/omission coverage rather than introducing a source-type +restriction accidentally. + +The initial 29-vector corpus was expanded to 47 before freezing. It includes +ordinary values, signed boundaries, one-outside rejections, sentinel collisions, +positive and negative precision boundaries and LONG_MIN+1. FLOAT/DOUBLE expected +results are literal IEEE bit patterns, not production casts in the oracle. +The architect independently checked all 21 non-null floating expectations using +BigInt significands, discarded-bit comparison and ties-to-even rounding; all +matched, without using Java or JavaScript floating-point casts. The +token denotes the target missing representation, explicitly SQL zero for +BYTE/SHORT. Canonical client resource: `io/questdb/client/cutlass/qwp/long-to-numeric.tsv`. +Frozen SHA-256: `0b574bc3b53fb7cc027b0cd2a9fc8d63f75a99d35f6f7f206d7bd21a70334f64`. + +Review also requires tests that distinguish completed-row preservation from +new-only-column rollback: the invalid row must introduce a separate column that +does not appear in either completed row, and its wire definition must disappear. +Duplicate coverage includes all six targets and both other existing setters, +not just same-setter repetition. Used extension parameters must be checked on a +newly supported type too. Range diagnostics name the target type as well as its +raw code and retain the stable non-retryable reason without echoing input values. + +Early focused runs failed because the test oracle conflated two valid INT null +encodings. Source LONG_NULL and explicit string(null) produce a null bitmap; +a non-null LONG equal to INT_MIN narrows to four sentinel bytes with no bitmap. +Both read as SQL NULL, but the wire assertion must distinguish them. The +architect traced `schema-numeric-focused.2.4d/focused.log` to that fixture error +and required an input-aware assertion while preserving correct production +conversion. Negative DOUBLE IEEE +goldens also require unsigned hexadecimal parsing, not Long.decode. Include +the vector case ID in failures so these distinctions are directly diagnosable. +The server E2E uses its public cursor's effective-null semantics and is not a +substitute for these exact client wire assertions. + +Client final-source focused verification passed 20 tests; log: +`core/target/schema-numeric-focused-final.2.4/focused.log`. The earlier +`schema-numeric-focused.2.4` through `2.4e` logs retain the failed fixture +assertions and unsigned-bit parsing error described above. Those fixture +corrections did not change production conversion behavior. + +Final standalone client regression and installation: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-acceptance.2.4 -Dmaven.javadoc.skip=true install +``` + +PASS: **306 unit tests plus two packaging checks**, zero failures/errors/skips. +Log: `core/target/schema-numeric-acceptance.2.4/focused-install.log`. +The architect verified aggregate results and matching workspace/installed +`questdb-client-1.3.10-SNAPSHOT.jar` SHA-256: +`847c13d172067281cae5149604c2585eb3392ae0028be48429cd4a93bace29e4`. +The client author also verified byte identity with cmp. Installed tests-JAR +SHA-256: `955dda25657168fb0375edc6d4a09bcb98a6e45af35a6e0bb639f8c70e7286ce`. +Three additional serial runs used `-Dtest=QwpSchemaRowBufferTest`, `surefire:test` +and `-Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-repeats.2.4/run-`. +Each passed all 20 tests. Logs: +`core/target/schema-numeric-repeats.2.4/run-1.log`, `run-2.log`, `run-3.log`. +Independent review accepted final client and server source; real-server runtime +validation is still required before accepting iteration 2.4. + +## D012 — Hold acceptance: null-sentinel parity depends on legacy block contents + +Date: 2026-09-09. Status: source and real-server confirmed compatibility issue; +the hold was resolved by the user decision in D013 on 2026-09-10. The evidence +below records why the original parity assumption could not be accepted. + +The first server runtime run also exposed the INT_MIN bitmap/sentinel assertion +mistake. Inspecting why the public cursor reported a present value led the +architect to a deeper issue: QwpFixedWidthColumnCursor.advanceRow only invokes +isCurrentValueSentinelNull when there is no null bitmap. D011's statement that +source LONG_NULL is always classified before conversion was incomplete. + +Ordinary QwpWebSocketSender.longColumn creates a nullable LONG column but calls +addLong with the raw value. An omitted row in the same column/block causes a +bitmap to be emitted. A supplied LONG_MIN value with its bitmap bit clear is +then treated as a present number: narrowing to BYTE/SHORT/INT fails its range +check; FLOAT/DOUBLE casts produce a finite large negative value. Without that +bitmap, the same supplied bits are treated as null. The LONG-to-LONG same-width +path still stores the LONG null sentinel. This is block-context-dependent +legacy behavior, not an ordinary value-only converter. + +Independent review confirmed the chain through Sender, QwpTableBuffer, +QwpFixedWidthColumnCursor and WalColumnarRowAppender. The 47-vector legacy +corpus contains supplied values but no omitted row in the same column, so it +does not establish parity for this discriminator. Existing LONG setter Javadoc +does not promise sentinel normalization, and the design's intentional omission +change does not by itself authorize changing a supplied LONG_MIN value. + +The architect recommends a fixed schema-mode rule: source null sentinels mean +null before buffering, independently of other rows. Preserve legacy client and +server behavior. Do not add block-dependent conversion, send-time transformation +or a server behavior change merely to make this test agree. This is an explicit +compatibility exception to supplied-value parity and needs user agreement before +the design is revised or iteration 2.4 is accepted. Current client code already +implements the proposed rule but remains a dormant component; ordinary Sender +activation is still absent. The server author is adding a real-server diagnostic +with raw LONG_MIN and omission in the same block; no server production edit is +authorized by this finding. + +Runtime confirmation: the fresh server gate initially passed 268 tests (the +previous 265 plus three numeric E2E tests); log +`core/target/schema-numeric-server.2.4/full-regression.log`. This was before the +mixed-bitmap discriminator and is not final acceptance evidence. Three added +diagnostic tests in QwpSchemaLongNumericE2ETest now intentionally fail the +previous parity assumption: + +- BYTE receives an out-of-range SCHEMA_MISMATCH NACK for the block. +- FLOAT stores `-9.223372E18` for supplied LONG_MIN, followed by NULL for omission. +- DOUBLE stores `-9.223372036854776E18`, followed by NULL for omission. + +Log: server `core/target/schema-numeric-server.2.4/legacy-min-bitmap-diagnostic.log`. +The architect verified all three failures and their actual server outcomes. +The diagnostic methods are `testLegacyLongMinWithBitmapNullToByteMatchesSchemaConversion`, +`testLegacyLongMinWithBitmapNullToFloatMatchesSchemaConversion` and +`testLegacyLongMinWithBitmapNullToDoubleMatchesSchemaConversion`. They preserve +the failed assumption visibly; do not silently weaken them while claiming +parity. Source is frozen, server seeded repeats stopped, and no server production +changes were made. Once the user chooses the contract, update these tests to +assert the agreed legacy/schema distinction or the agreed parity rule and rerun +the complete acceptance gate. The client 306+2 gate and artifact remain valid +evidence for the proposed component, not a claim that the product contract is met. + +## D013 — Approved: consistent LONG null handling in schema mode + +Date: 2026-09-10. Status: user approved; accepted and verified in iteration 2.4. + +The user agreed that adding an empty row should not change another row's value +and accepted the recommendation that the new client should not reproduce the +server inconsistency. Adopt the previously proposed rule: for supported numeric +targets, LONG_MIN is source null before buffering, independent of block contents. +Legacy behavior remains unchanged. This approval is not authorization for a +separate legacy server fix or ordinary Sender activation. + +Design revision 15 records the rule as an explicit exception to supplied-value +conversion parity. Target missing representation remains zero for BYTE/SHORT +and NULL for INT/LONG/FLOAT/DOUBLE. Duplicate-first behavior and input/target +support validation precede normalization; LONG_NULL must not enable an otherwise +unsupported pair. The already-reviewed client component implements this rule; +no additional production change is needed to resolve the hold. + +Replace the three failed parity assumptions with deliberate compatibility tests. +They must separately prove the preserved legacy outcomes and the stable schema +outcomes with and without omission in the same column/block. Include the full +six-target numeric family, exact wire representations and independent SQL +expectations. Preserve the failed diagnostic log as evidence of the decision; +do not describe the resulting green tests as full legacy parity. + +SOL server author owns those test changes and the fresh real-server gate; +SOL client author revalidates the unchanged artifact and client gate; independent +SOL reviewer challenges the contract, source and assertions. The architect owns +the design, journal and final acceptance. Keep per-repository Maven runs +serialized, use the installed standalone client, and preserve unrelated work. +No server production changes, send-time transformation, extra mode switch or +per-row allocation mechanism are introduced. A legacy-server bug fix remains +separate work with its own compatibility review. + +## Iteration 2.4 acceptance — approved null contract + +Date: 2026-09-10. Status: accepted. The D012 hold is closed by D013 and the +fresh checks below, not by claiming full legacy conversion parity. + +The final server test replaces the three false-parity diagnostics with +`testLongMinSchemaNullIsStableWhileLegacyDependsOnBlockBitmap`. It covers all +six numeric targets, each with and without an omitted row, separately for +legacy and schema encoding. Wire assertions retain legacy TYPE_LONG and raw +LONG_MIN bits, prove when a bitmap exists and whether the value is present, +and contrast schema encoding's target type and explicit missing-value bit. +SQL assertions independently verify stable schema missing values and the +preserved legacy range errors or finite FLOAT/DOUBLE values. The existing +47-vector corpus, partial-row rollback, duplicate-first behavior and explicit +null/omission checks remain in the gate. The failed D012 log is retained. + +Fresh client verification, from the standalone client repository: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-numeric-final-verify.2.4 -Dmaven.javadoc.skip=true verify +``` + +PASS: **306 selected unit tests plus two packaging checks**, zero +failures/errors/skips. Log: +`core/target/schema-numeric-final-verify.2.4/focused-verify.log`. +No client source edits or reinstall were needed for this decision. + +Fresh real-server gate, from the server repository, without `local-client`: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaIdentityParserTest,QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaLongNumericE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide,QwpUdpMalformedTest,QwpUdpInsertTest' -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-numeric-server.2.4 test +``` + +PASS: **269 selected tests**, zero failures/errors/skips. Log: +`core/target/schema-numeric-server.2.4/final-regression.log`. +The same command with only `-Dtest=QwpSchemaLongNumericE2ETest` first passed +all four numeric tests; log: +`core/target/schema-numeric-server.2.4/contract-focused-wire.log`. + +Three additional serial server runs used +`-Dtest=QwpSchemaLongNumericE2ETest,QwpSchemaRowBufferE2ETest`, `surefire:test`, +the same absolute temporary directory, and seed pairs supplied through +`-Dfuzz.s0`/`-Dfuzz.s1`: `(1,2)`, `(42,43)`, `(1234567,7654321)`. +Each passed **eight tests**, zero failures/errors/skips. Logs under +`core/target/schema-numeric-server.2.4/repeats/`: +`repeat-1-2.log`, `repeat-42-43.log`, `repeat-1234567-7654321.log`. + +The architect rechecked aggregate logs, final wire assertions and the Surefire +XML classpath, which uses the installed standalone client from +`/home/jara/.m2/repository/org/questdb/questdb-client/1.3.10-SNAPSHOT/`. +Workspace and installed main JARs retain matching SHA-256 +`847c13d172067281cae5149604c2585eb3392ae0028be48429cd4a93bace29e4`. +Both repositories' shared corpus retains SHA-256 +`0b574bc3b53fb7cc027b0cd2a9fc8d63f75a99d35f6f7f206d7bd21a70334f64`. +Independent SOL review accepted revision 15, final source and runtime evidence +with no remaining blocker. Both repositories pass `git diff --check`. + +Reflection: a corpus of individual supplied values was insufficient to prove +legacy parity because missing values elsewhere in a block change decoding. +Future conversion slices must vary omission/explicit-null context as well as +individual boundary values. Keep exceptions explicit in the contract instead +of letting batching determine conversion results. + +This completes the bounded LONG-input numeric component iteration. Ordinary +Sender setters remain unactivated; no legacy server production behavior, +send-time transformation or mode switch changed. No commits were made. +The next conversion family remains separate work requiring its own bounded +brief, source validation and real-server conformance tests. + +## D014 — Record integration simplification opportunities + +Date: 2026-09-10. Status: recorded at the user's request; implementation unchanged. + +Design revision 16 adds a separate "Simplification opportunities" section after +the progress reflection: reuse Sender-owned row storage and lifecycle, keep +lookups with the existing I/O owner, share schema metadata/payload machinery, +and make current implementation status easy to find. These are integration +guidelines, not completed refactors or authorization for partial activation. +Pinned snapshots, compatibility, bounded requests and delivery rules remain +unchanged. The implementation-status checklist is recorded as an opportunity; +this documentation change does not add it or select the next implementation +slice. + +## D015 — Schema feedback before further conversion families + +Date: 2026-09-10. Status: accepted. The fresh SOL review reopened the initial +acceptance; D016 records the correction, independent review and final regression +evidence that restored it. Wire contract and scope remain unchanged. + +Following the progress reflection, the user asked to continue. Prioritize the +schema-update path over another isolated converter family. This bounded slice +adds real server ACK/NACK feedback and a production client decoder, exercised +with the existing conversion component. Ordinary Sender activation, latest- +schema cache ownership/adoption, lookup coordination and automatic row-boundary +transitions remain future work. Do not add an unused callback/cache framework +or activate only selected setters to disguise those gaps. + +Root and three SOL agents inspected response parsing, metadata discovery, +cumulative ACKs, deferred commit groups, partial sends and schema-required SF +replay before freezing the contract. Current table identities are parsed and +persisted but are not consumed by ingestion; this slice gives them a feedback +purpose, never a version precondition. Negotiated legacy blocks also have an +unknown identity and can receive feedback. + +Revision 17 defines status mode bits: `0x00` unchanged response, `0x80` complete +update list, `0xC0` invalidate all future lookup-cache entries, `0x40` reserved. +Low six bits retain the ordinary status. Feedback requires negotiation; +durable ACKs never carry it. Named, length-delimited entries reuse the SCHEMA +payload with request ID zero reserved for write feedback. DESCRIBE remains +positive-ID. Full validation precedes ACK/NACK handling and metadata exposure. + +Simplification chosen after adversarial review: if the complete metadata set +cannot fit the 1 MiB UPDATES-payload and raw-response-buffer bounds, send +invalidation instead of paginating or repeating cumulative ACKs. Encoding the +mode in existing status bits makes invalidation exactly the base response's +length. This avoids even a one-byte capacity regression and requires no new +send continuation. It affects future lookup cache entries only, never pinned +rows, persisted bytes, delivery watermarks or terminal-error policy. + +Collect bounded names/identities; obtain coherent, freshly authorized snapshots +when serializing responses. Keep successful ACK-prefix feedback separate from +rejected-frame/deferred-group feedback. Preserve both across backpressure under +the existing response ownership; clear only the fully sent set. NACK metadata +may cover authorized tables encountered in rejected work, not unrelated tables +or metadata owned by a pending earlier ACK. Unavailable/unauthorized complete +feedback may use nameless invalidation without changing the data outcome. + +Assignments: + +- SOL server author: scoped server metadata/feedback state and wire writes, + real-server feedback/evolution, authorization, bounds, ACK ordering and + partial-send tests. No WAL conversion or UDP behavior change. +- SOL client author: shared schema payload codec, negotiation-aware response + decoding, existing foreground/background replay consumers, malformed-response + and delivery-regression tests. No alternate Sender API or unused observer. +- Independent SOL reviewer: adversarial wire, ownership, bounds, compatibility + and final source/evidence review, read-only and separate from authors. +- Architect: design, journal, scope, source assumptions and final acceptance. + +Per-repository builds remain serialized. Use the installed standalone client, +never the server's `local-client` profile, and verify artifact identity. Preserve +all existing dirty work and the unrelated server Rust checkout. This is +functional protocol work, not a performance optimization: no bottleneck or +speedup claim, speculative tuning or new per-row conversion work is authorized. +Performance remains unmeasured and must not be inferred from test time; +performance optimization would require the hardware-first measurement gate. + +Pre-acceptance clarification: root challenged applying the 1 MiB limit to +INVALIDATE_ALL. An existing base ACK can exceed 1 MiB when the configured +response buffer and table set are large. The new limit therefore applies to +the complete UPDATES payload only; NONE and INVALIDATE_ALL keep existing base +bounds. Independent review agreed. Add paired codec checks for a large valid +base ACK and its same-length invalidation form, while rejecting oversized +UPDATES. This preserves the fallback's zero-added-size guarantee without +introducing a global receive cap. + +## Iteration 2.5 development evidence + +Source baselines remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and +server `12a33d651e51e2682e7a448c8db5168fc72dfad3`, plus the preserved dirty +changes from accepted iterations. Root verified both HEADs and the installed +artifact used below. All builds use repository-local temporary storage because +the shared `/tmp` has little space; unrelated files are not removed. + +Client pre-edit response/replay/durable baseline passed 41 tests: +`core/target/schema-feedback-baseline.2.5/baseline.log`. The first focused run +failed the old unknown-status fixture using `0xFF`, which is now an explicitly +negotiated response mode rather than an unqualified legacy error. The fixture +now uses unknown base status `0x3F`; separate tests reject unnegotiated modes. +This is a protocol-reservation correction, not removal of unknown-base-status +coverage. Failed log: `core/target/schema-feedback-focused.2.5/focused.log`. +The subsequent focused gate passed 58 tests: +`core/target/schema-feedback-focused3.2.5/focused.log`. + +The initial standalone install passed 331 selected tests plus two packaging +checks; log `core/target/schema-feedback-install.2.5/install.log`. Workspace +and installed main JARs match SHA-256 +`e9e9d93f63a1a9a25b33163a6db4a027152bb8c174030b965450bb64211e6825`. +This supplies the server with the production decoder. Network tests and final +source validation were still being extended afterward; this is not iteration +acceptance evidence by itself. + +Construction review found and required corrections before acceptance: + +- A reset block had landed in `onAckBlocked`, clearing rejected/deferred + metadata before copying ACK ownership. Keep connection reset separate from + the parked ACK snapshot; test a blocked ACK followed by a distinct NACK. +- Table-writer acquisition can fail before append, for example for a non-WAL + table. Capture encountered-table feedback on that path too, without masking + the original failure or adding metadata reads to matching successful writes. +- Preflight declared update count against available minimum entry bytes before + allocation; validate table names with server-compatible case-insensitive + duplicate checks and existing name rules. +- A size-limit rejection test must contain an otherwise complete valid UPDATES + response; a missing trailer would reject independently of the limit. Use + valid, distinct table names in the large base-response compatibility fixture. +- WebSocket short-payload capacity must clamp at 125 bytes for 128/129-byte + buffers, before the larger header becomes necessary. + +These are in-progress review findings, not defects in an accepted iteration. +Final runtime gates and independent stable-source review remain required. + +Client canonical final-source verification passed 333 selected tests plus two +packaging checks; log `core/target/schema-feedback-canonical.2.5/verify.log`. +Three serial runs of `QwpSchemaFeedbackResponseTest,QwpSchemaReplayNetworkTest` +each passed 15 tests; logs +`core/target/schema-feedback-repeats.2.5/run-{1,2,3}/test.log`. +The production artifact hash remains `e9e9d93f...`; no reinstall occurred while +the server used it. Independent client review accepted source and evidence. +The malformed-response network test uses an observable second replay attempt +and bounded close configuration, then reopens public SF state to verify +`ackedFsn=-1`, retained published frames and no quarantine. Earlier unbounded +test-harness attempts are not counted as successful validation. + +Initial real-server checks passed five identity E2Es and two new feedback E2Es; +logs `core/target/schema-feedback-server.2.5/identity-test.log` and +`feedback-e2e.log`. The exact-capacity test sends a 252-byte base ACK plus its +four-byte WebSocket header through a 256-byte response buffer, preserving both +table transaction entries and SQL rows with same-length invalidation. The +pre-writer non-WAL failure returns authorized current metadata and no rows. +These are partial gates, not final server acceptance. + +Independent server review also found that resumed NACK completion did not +release its metadata ownership like immediate completion; this was corrected. +Root challenged a proposed later-unrelated-NACK test: the existing unresolved +sequence guard prevents that ordinary data sequence on the same connection. +The reviewer agreed to test reachable partial-NACK completion and retained +delivery behavior, not invent a private-state test or claim a demonstrated leak. + +After the client gate completed, the client SOL author took ownership of the +new server `QwpSchemaFeedbackAuthorizationE2ETest.java` only, in parallel with +the server author's remaining lifecycle tests. Server production and Maven +remain solely server-author-owned; the independent reviewer remains read-only. + +The client author subsequently added a separate evolution E2E and handed both +test files to the server author before combined compilation. Initial combined +runs exposed fixture errors: invalid WAL DDL, duplicate frame-handler methods, +using a wire type constant for a metadata type, and an old response helper that +did not pass negotiated capability. These failures are preserved in the +`feedback-matrix-*.log` files, not counted as successful gates. Root also +required ordered DESCRIBE barriers before revocation or ALTER, rather than +assuming that sending DATA proves server processing. Native encoder pointers +must be read after encoding, which can grow the underlying buffer. + +Revocation tests corrected a load-bearing assumption: the existing commit path +checks INSERT authorization again. Revoking it after deferred ingestion can +produce a SECURITY_ERROR NACK, not a successful ACK. Test the existing data +outcome and nameless metadata invalidation; do not change authorization to make +the original fixture expectation pass. + +Root challenged the claim that the existing live-table limit also bounded +pending feedback. A proposed rowless deferred/drop/eviction sequence appeared +to allow historical names to accumulate, and root temporarily authorized an +explicit bounded accumulator with an invalidation marker. Both author and +reviewer initially accepted the counterexample. Further source tracing then +disproved its premise on the WebSocket path: the independent stale-entry scan +is UDP-only. WebSocket lookup evicts under the same requested name, while a new +distinct name hits the table ceiling; rename and buffered-drop eviction paths +reject the frame. The captured table token also does not silently follow pure +renames. No unbounded WebSocket sequence was established. + +Correction and final decision: withdraw that blocker and the proposed new +accumulator. Restore the simpler direct maps, bounded by the configured table +ceiling plus at most the terminal failing name. Document this source dependency +and revisit it if cache eviction changes. Do not add an invalidation latch or +new test obligations merely to defend the earlier hypothesis. The temporary +helper compiled but is not part of the accepted change. This corrects root's +earlier user-facing claim of a reachable gap as well. + +One independently verified simplification remains: remove the redundant parked +ACK metadata copy. The existing raw send buffer already owns the exact serialized +bytes; resuming that send needs no retained or resampled schema objects. Keep +pending rejected-work metadata separate until its NACK is serialized. Final +server runtime and review acceptance remain pending. + +The corrected server component matrix passed 26 tests in +`core/target/schema-feedback-server.2.5/feedback-matrix-7.log`. In particular, +the mixed authorization test preserves the existing partial-commit outcome: +the earlier authorized table retains its row, the denied table has none, and +the NACK exposes no named schema. This is not an all-or-nothing transaction +guarantee. The evolution E2E verifies INT bytes encoded before ALTER, LONG +metadata in the stale write's ACK, byte-identical re-encoding from the old +component, and a subsequent LONG wire value above INT_MAX with no redundant +feedback. The combined gate still precedes final cleanup and broad regression. + +Canonical client verification command (client repository; selected tests, not +the entire project suite): + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,QwpSchemaFeedbackResponseTest,WebSocketResponseTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaRowBufferTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-feedback-canonical.2.5 -Dmaven.javadoc.skip=true verify +``` + +Each client repeat uses the same command shape with +`-Dtest=QwpSchemaFeedbackResponseTest,QwpSchemaReplayNetworkTest`, `test` rather +than `verify`, and its own absolute temporary directory under +`core/target/schema-feedback-repeats.2.5/run-{1,2,3}`. + +The first broad server gate ran 280 tests with two failures, both in existing +LONG-numeric E2Es whose response helper negotiated schema but still invoked +the legacy decoder overload. Negotiated legacy table blocks correctly return +feedback for their unknown identity. Correct the test helper to pass connection +capability; do not suppress production feedback or weaken SQL/wire assertions. +Failed gate preserved as `core/target/schema-feedback-server.2.5/final-canonical.log`. +The other 278 tests, including all new feedback cases, passed; the complete +corrected gate and seeded repetitions are still required. + +## Iteration 2.5 final acceptance — 2026-09-10 + +Accepted by the architect after the independent SOL review returned ACCEPT and +both authors froze their source. No remaining source or targeted-runtime blocker. +This accepts the bounded feedback slice, not automatic schema-mode Sender. + +- Client canonical gate: **333 selected tests plus two packaging checks**, zero + failures/errors/skips. The seven network replay tests are included in 333, + not an additional count. Three serial feedback/replay repeats passed **15 + tests each**. Logs and command are recorded above. +- Server canonical gate: **280 selected tests**, zero failures/errors/skips. + Log: `core/target/schema-feedback-server.2.5/final-canonical-2.log`. +- Server focused repeats: **19 tests each**, zero failures/errors/skips, with + seeds `(1,2)`, `(42,43)` and `(1234567,7654321)`. Logs: + `core/target/schema-feedback-server.2.5/repeats/repeat-1-2.log`, + `repeat-42-43.log` and `repeat-1234567-7654321.log`. +- Root reverified the matching workspace/installed client JAR SHA-256 + `e9e9d93f63a1a9a25b33163a6db4a027152bb8c174030b965450bb64211e6825`. + Surefire XML points to the installed standalone `.m2` artifact, not + `local-client`, and confirms the final seed pair and repository-local temp + directory. Both repositories pass `git diff --check`. + +Canonical server command (server repository): + +```sh +mvn -B -pl core -DargLine='-ea -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/home/jara/devel/oss/questdb/core/target/schema-feedback-server.2.5/tmp -XX:+UseParallelGC --sun-misc-unsafe-memory-access=allow --enable-native-access=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.time.zone=ALL-UNNAMED --add-exports=java.base/jdk.internal.vm=ALL-UNNAMED --add-exports=java.base/jdk.internal.vm=io.questdb' '-Dtest=QwpSchemaIdentityParserTest,QwpSchemaIdentityE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaLongNumericE2ETest,QwpSchemaDiscoveryE2ETest,QwpSchemaDiscoveryAuthorizationE2ETest,QwpSchemaDiscoveryLimitsE2ETest,QwpSchemaFeedbackE2ETest,QwpSchemaFeedbackAuthorizationE2ETest,QwpSchemaFeedbackEvolutionE2ETest,QwpWebSocketProtocolTest,QwpUpgradeRejectFragmentationTest,QwpServerCloseDrainTest,QwpAckSeqTxnCoverageBlackBoxTest,QwpWebSocketTypeConversionE2ETest,QwpSenderE2ETest#testTypedUuidPreservesCompletedRowAfterLocalError+testUuid,QwpWebSocketSenderReceiverTest#testColumnTypeMismatchThrowsClientSide,QwpUdpMalformedTest,QwpUdpInsertTest' test +``` + +Server repetitions reuse the identical `-DargLine`, replace the selection with +`-Dtest=QwpSchemaFeedbackE2ETest,QwpSchemaFeedbackAuthorizationE2ETest,QwpSchemaFeedbackEvolutionE2ETest,QwpSchemaRowBufferE2ETest,QwpSchemaLongNumericE2ETest`, +use `surefire:test`, and add `-Dfuzz.s0=1 -Dfuzz.s1=2`, then `42/43`, then +`1234567/7654321`, serially. Passing `java.io.tmpdir` in the fork's JVM arguments +is important: earlier server runs still used `/tmp` despite a Maven-level +property. Final XML verifies the intended location. No unrelated temporary +files were deleted. + +Delivered behavior is demonstrated through real negotiated writes and observable +wire/SQL outcomes: matching identities omit feedback; stale/unknown identities +return current authorized metadata; oversized or denied complete feedback uses +nameless invalidation; deferred groups coalesce names; describes and subsequent +ACKs agree after ALTER; partial sends preserve separate ACK/NACK contents; old +INT bytes remain unchanged while a later component encodes LONG. Malformed +feedback cannot release persisted replay data. Metadata does not change ordinary +or durable ACK coverage, rejection policy, or partial-commit semantics. + +Simplicity decisions retained: shared schema payload codec, one complete update +set or same-length invalidation, response-time snapshots, and no duplicate +metadata owner for an already serialized ACK. The speculative accumulator was +removed, and its disproved motivating sequence is recorded above rather than +presented as a fixed defect. + +Remaining work: latest-schema cache ownership/adoption, lookup coordination, +automatic row-boundary transitions, the remaining conversion families and +coherent activation in ordinary Sender. The next architecture slice should +resolve producer-owned row/block integration using the existing buffer owner; +do not grow another independent row store or activate a subset of setters. +Tests are a targeted regression matrix, not exhaustive proof of every existing +close/durable permutation. Performance is unmeasured; no throughput or latency +claim follows from these functional checks. No commits or pushes were made; +unrelated work, including the server Rust checkout, was preserved. + +## Fresh independent SOL review — 2026-09-10 + +At the user's request, a new SOL agent reviewed iteration 2.5 independently, +without the implementation conversation. It inspected the design, current +client/server source, tests and recorded execution evidence. Root separately +reverified artifact identity, test counts, seeds and temporary paths. No Maven +rerun, implementation edit or commit was performed during this review. + +Verdict: reopen iteration 2.5 acceptance for one **P2** finding. Root agrees. +`QwpSchemaProtocol.decodePayload` checks column-name length and UTF-8 but does +not apply column-name rules or reject case-insensitive duplicates. A schema +containing `x` and `X`, or an illegal name such as `a/b`, can therefore pass the +shared decoder. `WebSocketResponse.readFeedback` trusts that decoded schema; +the successful ACK path can then advance replay progress. Duplicate detection +in `QwpSchemaRowBuffer` occurs later and does not protect the ACK boundary. + +The reviewer reproduced duplicate-column acceptance against the verified +installed client JAR: `decoded=2 names=x,X`. Acceptance of the enclosing ACK and +the replay-watermark consequence were traced in source, not reproduced through +a network replay. No data-loss incident or malformed response from the current +server was demonstrated. The issue is the promised rejection of malformed +metadata before delivery progress; existing replay tests cover malformed +framing, not these invalid column definitions. + +Recommended correction, not implemented in this review: validate server-compatible +column names and case-insensitive uniqueness in the shared schema decoder. Add +DESCRIBE and ACK-feedback rejection tests plus a public replay-network case +showing the malformed schema leaves `ackedFsn == -1`. Rerun the affected gates +before restoring acceptance. + +No other blocking defect was found. A suspected combined-response 1 MiB sizing +mismatch was disproved: the caller subtracts the base response from the capped +complete-payload budget before encoding feedback. Missing-table create-permission +reporting was not promoted to a defect: MISSING reports absence, not permission +to create, and actual creation still enforces authorization. Automatic cache +adoption and ordinary Sender row transitions remain outside the reviewed slice. + +## D016 — Validate schema column names at the decoding boundary + +Date: 2026-09-10. Status: accepted and verified; iteration 2.5 acceptance restored. +The user asked to fix the P2 without extra complexity. + +Put column-name validity and case-insensitive uniqueness checks in the existing +shared decoder loop. Reuse `TableUtils` and an existing bounded hash collection; +do not add a validator framework, configuration, wire field, lookup state or +quadratic scan. Both DESCRIBE and write feedback then share the same rejection +boundary. This is schema-metadata work, not per-row conversion validation. + +Root verified that `QwpSchemaResponse` is constructed only by the decoder and +its constructor is package-private. Once decoding guarantees valid unique names, +remove the later duplicate-error branch from `QwpSchemaRowBuffer`; retain the +name-to-column binding map it actually needs. Do not retain another name index +in the immutable schema just to avoid a short-lived decoder collection. + +The new SOL author owns the two scoped client production files and existing +codec/feedback/replay tests. The fresh SOL reviewer remains independent and +read-only. Root owns architecture, journal and acceptance. Preserve all earlier +dirty work and failed-run evidence; no commits, ordinary Sender activation, +other converters or server implementation changes are part of this fix. + +Require failing tests on the old decoder, then a green focused gate: direct +DESCRIBE and ACK/NACK feedback reject duplicate/illegal column names before +publishing any metadata. Preserve valid Unicode, spaces and boundary-length +names. Reuse the public replay-network harness to show semantically malformed +feedback retains the published backlog with `ackedFsn == -1`, rather than just +asserting a parser exception. No reflection, private-state hooks or fixed sleeps. +Revalidate the client aggregate and real-server matrix with the newly installed +standalone artifact before restoring iteration 2.5 acceptance. Tests remain +functional evidence; no performance claim is made. + +D016 construction evidence: the initial focused red run failed the new direct +DESCRIBE and feedback checks, but the new network retention test still passed +with the old decoder. Root traced this to the existing feedback test helper: +its latch counted two replies, while the seeded backlog contains two frames. +Both replies could be sent on the first connection, followed by immediate +sender close before either ACK was processed. The earlier description of this +as an observable second replay attempt was therefore too strong. + +Require replies on distinct peer connections before closing the sender. This +observes an actual reconnect without production hooks, and strengthens both +the existing truncated-feedback case and the new invalid-column case. Verify +that the corrected semantic network test fails against the previous installed +artifact, without restoring production source to manufacture the red run. +This is test coordination, not additional production state. + +The first post-fix focused run also failed native library extraction because +the configured JVM temporary directory did not exist (`File.createTempFile` +reported ENOENT). Creating the scoped directory corrected the environment; +no native or production-code change was required. Preserve that failed log +alongside the rerun; it is not acceptance evidence. + +The first completed aggregate run found one stale row-buffer test expecting the +later duplicate-name exception. Remove that redundant test: the decoder tests +now own this invariant, and invalid schemas cannot reach the row buffer. +This changes the final selected client count from 337 to 336 without dropping +coverage. An earlier incomplete aggregate log is also retained and is not +counted as a passing gate. + +### D016 final verification and acceptance + +The production change is confined to the shared schema decoder and removal of +the redundant row-buffer check. The decoder reuses the existing +`LowerCaseCharSequenceIntHashMap`, bounded by the existing 2,048-column limit. +Its case comparison and `TableUtils` name rules match the server. There is no +new production type, protocol field, setting or retained state. The row buffer +still has its required binding map. Server source is unchanged by D016. + +The final public-Sender network test fails against the previous installed client +JAR, SHA-256 +`e9e9d93f63a1a9a25b33163a6db4a027152bb8c174030b965450bb64211e6825`. +The old JAR precedes compiled test classes on the classpath. It accepts the +malformed feedback and stays on one connection; the test fails specifically at +`invalid schema feedback must be rejected and replayed`, not native startup. +With the fix, two distinct peer connections are observed and reopening the +published backlog verifies `ackedFsn == -1`, `publishedFsn == 1` and no quarantine. +The direct codec and whole-ACK/NACK tests also reject illegal names and exact, +ASCII-case and Unicode-case duplicates, while retaining valid name boundaries. + +Final evidence (all successful gates have zero failures, errors and skips): + +- Client: **336 selected tests and two packaging checks** in + `core/target/schema-feedback-name-fix/canonical-pass.log`. +- Client feedback/replay repeats: **17 tests each, three runs**, in the same + directory's `repeat-1.log`, `repeat-2.log` and `repeat-3.log`. +- Old-artifact negative proof: `final-network-old-installed-red.log`; initial + focused codec/feedback negative proof: `focused-red.log`. Failed environment + runs and the stale-test aggregate failure remain separate from acceptance. +- Server: **280 selected tests** in + `core/target/schema-feedback-name-fix-server/canonical.log`. +- Server feedback/conversion repeats: **19 tests each**, seeds `1/2`, `42/43` + and `1234567/7654321`, in that directory's corresponding `repeat-*.log` files. + +The installed standalone client and workspace JAR have identical SHA-256: +`9eb4c6716a3dd65c8c4bf6e2de92427f80a7c7a2dc4840aadfbdb75322064fa0`. +Root verified this before and after the server runs, and verified the installed +JAR on the server test classpath. The server did not use `-Plocal-client`. +Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD remains +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with the ongoing task's uncommitted +changes. Builds were serialized per repository. Test JVMs used existing, +absolute repository-local temporary directories, including forked JVM options. + +Exact commands, including the old-artifact classpath reference and per-run JVM +temporary paths, are recorded in the local +[run manifest](../core/target/schema-feedback-name-fix/run-manifest.md). Client +and server test selections match D015; the client aggregate includes the new +tests and removal of the redundant row-buffer test described above. + +The independent SOL reviewer accepted the final source, client and server +evidence. Root independently verified the gates and artifact provenance before +restoring acceptance. Both repositories pass `git diff --check`. This closes +the P2, not the remaining latest-schema adoption or ordinary Sender activation +work. No commits, pushes or performance claims were made. + +## D017 — Separate schema conversion from buffer ownership + +Date: 2026-09-10. Status: accepted and verified; iteration 2.6. + +The user approved the next integration step. Root's preceding recommendation +overstated how soon automatic INT-to-LONG adoption could be exercised through +ordinary Sender. Read-only SOL review and root's source trace found no +production consumer for decoded schema updates, no first-use DESCRIBE +coordinator, and no complete Sender conversion/inference integration. Starting +with a legacy probe and adopting feedback would bypass required first-use +lookup and constitute the prohibited partial activation. Do not manufacture +that E2E by adding an opt-in or test-only Sender mode. + +Deliver the ownership prerequisite first: make the conversion component bind +to an existing `QwpTableBuffer`, without allocating, closing, resetting, +completing or encoding a second row store. Keep one owner for buffer lifetime, +row completion, cancellation, byte/row accounting and encoding. Preserve the +existing UUID and LONG numeric conversion contracts and real-server coverage. +No borrowing/owning flag, alternate buffer queue, cache scaffold or ordinary +Sender activation belongs in this increment. Pin schema identity to the bytes +at the owner boundary; a borrowed converter must not provide a way to mix two +snapshots in a non-empty buffer. + +Before editing, root read the hardware-first-performance skill and its required +USE/hardware references; SOL performed a read-only capability preflight. There +is no existing benchmark for the schema component's fixed-work write/encode +path. The available Sender latency benchmark exercises a different, non-schema +path, and the host's current system-wide I/O pressure needs workload-correlated +analysis before any resource classification. The skill's optimization study +cannot be applied cleanly to this bounded functional task without creating a +separate measurement project. Fallback: no performance optimization or claimed +bottleneck/speedup, no change to ordinary Sender's row setters, source review of +ownership/allocation changes, and functional regression evidence only. Encoding +entry points gain guards for newly schema-bound buffers; do not describe that +as zero code change on the existing encoding path or as measured zero overhead. + +Root owns this brief, documentation and acceptance; SOL implementation and an +independent SOL review remain separate. Preserve all prior dirty work, serialize +builds per repository, use existing absolute repository-local JVM temp paths, +install the standalone client before real-server validation, and never use +the server's `-Plocal-client` profile. No commits or pushes are authorized. + +D017 API decision: replace the unreleased owning `QwpSchemaRowBuffer` with +`QwpSchemaBinding(QwpTableBuffer, QwpSchemaResponse)`. The owner creates and +closes the table buffer, completes rows, and cancels/rolls back a partial row +when a setter fails. The binding only resolves, converts and appends values. +Do not retain a compatibility wrapper that owns another buffer lifecycle. + +The table buffer holds its current binding reference. Attach only to a wholly +empty, unbound layout, after validating/constructing the binding. `reset()` +preserves the attachment and column definitions; `clear()` detaches before +cleanup. Each binding setter checks that it is still current before touching +the buffer. Root rejected the proposed separate layout-lease object and +generation counter: reference identity already detects stale bindings, and the +binding already holds the immutable schema. Do not duplicate schema identities +or introduce a reattachment API for previously invalidated bindings. + +Schema encoding derives identity from the buffer's binding. Existing explicit +identity entry points remain valid for unbound protocol buffers and bound +buffers with the exact pinned identity; reject mismatches before changing the +encoder. Legacy encoding rejects a bound buffer rather than stripping its pin. +For bound buffers, partial rows cannot be encoded; an empty single-table encode +returns zero without modifying the encoder, while appending an empty bound table +to a predeclared multi-table frame is rejected. Existing unbound-buffer encoding +contracts remain unchanged. These are owner-misuse errors, not value-conversion +failures or reasons to cancel another row automatically. + +Real-server tests must use this explicit owner lifecycle and retain their SQL, +wire and shared-vector assertions. Evolve a caller-owned buffer from INT to +LONG only after retaining its old encoded bytes, clearing its old layout and +binding the new schema. This proves the ownership prerequisite and immutable +old bytes; it is explicitly not automatic feedback adoption by Sender. + +### Iteration 2.6 development and review evidence + +The first migrated client gate ran 19 tests and found three callers still +assuming the old facade's automatic rollback. The replacement binding correctly +left cancellation to the owner; the tests needed explicit owner cancellation +and uncommitted-column rollback. Keep their original row/value expectations, +and rename the key regression to describe owner rollback. Preserve the failed +run as `core/target/schema-binding-2.6/logs/focused.log`, not acceptance evidence. +The completed focused suite has 23 tests, including reference-based stale +binding rejection for all three setters while a newer row remains intact, +same-snapshot clear/rebind, reset/layout retention, failed-constructor reuse, +and byte-level encoder checks for bound-buffer framing, identity and partial/ +empty-row errors. + +The independent SOL reviewer accepted the reference-identity attachment and +verified the final guard ordering. Root required rejection before the +`encodeWithDeltaDict` convenience method starts a new frame, and preserved +matching explicit-identity encoding as byte-identical to derived encoding. +No lease object, counter, additional schema representation or owning adapter +was needed. Root also required explicit owner cleanup in failed-constructor +tests and verified that the same buffer accepts a valid binding afterward. + +Three existing real-server E2E classes were migrated without changing server +production code. The first build stopped at test compilation because a renamed +buffer local shadowed a table-cursor local; no tests ran. The corrected focused +gate passed nine tests. Root restored the pre-existing old-frame re-encoding +equality assertion before clear/rebind and strengthened wire assertions to +check exact old/new table identities, not merely non-negative identities. +SQL, UUID/numeric corpus, null semantics and completed-row preservation checks +remain intact. These tests still exercise the production components over a real +connection; they do not pretend to exercise automatic ordinary Sender adoption. + +An incremental-build rename can leave obsolete classes in the artifact. Root +required source, compiled-output and JAR checks for the removed owning facade. +No stale compiled files were found, so no cleanup or quarantine was needed. +Earlier logs were preserved, and neither repository was blanket-cleaned. + +Design revision 18 adds a short current implementation-status section, as +suggested by D014, so the intended contract cannot be mistaken for a completed +feature. Detailed history and test evidence remain here. + +### Iteration 2.6 final acceptance — 2026-09-10 + +Accepted by root following independent SOL source/adversarial review and +completed client/server gates. This accepts the non-owning binding prerequisite, +not automatic schema adoption or normal Sender activation. + +- Client final canonical gate: **340 selected tests plus two packaging checks**, + zero failures/errors/skips. The 23 binding tests are included in 340. Log: + `core/target/schema-binding-2.6/logs/canonical-final.log`. +- Server focused migrated E2Es: **9 tests**, then canonical **280 tests**, all + green. Logs: `core/target/schema-binding.2.6-server/focused3-rerun.log` and + `canonical280.log` in the server repository. +- Three server feedback/conversion repeats: **19 tests each**, seeds `1/2`, + `42/43`, `1234567/7654321`, zero failures/errors/skips. Logs in the same server + directory: `repeat-1-2.log`, `repeat-42-43.log`, + `repeat-1234567-7654321.log`. + +Client canonical command, run from the client repository: + +```sh +mvn -B -pl core '-Dtest=QwpSchemaProtocolTest,QwpSchemaFeedbackResponseTest,WebSocketResponseTest,WebSocketClientSchemaNegotiationTest,WebSocketClientTest,WebSocketClient421RoleHeaderTest,QwpWebSocketEncoderTest,QwpTableBufferTest,QwpWebSocketSenderTest,QwpWebSocketSenderMultiEndpointTest,QwpQueryClientUpgradeStatusTest,QwpSchemaBindingTest,RecoveredFrameAnalysisTest,CursorWebSocketSendLoopOrphanTailTest,BackgroundDrainerOrphanTailTest,CursorWebSocketSendLoopForegroundReconnectPolicyTest,CursorWebSocketSendLoopReconnectLeakTest,QwpSchemaReplayNetworkTest,CursorWebSocketSendLoopDurableAckTest,BackgroundDrainerDurableAckRetryTest' -Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-binding-2.6/tmp -Dmaven.javadoc.skip=true verify +``` + +The focused client command selects `QwpSchemaBindingTest` and uses `test` with +the same temporary path; install used +`mvn -B -pl core -DskipTests -Dmaven.javadoc.skip=true install` after the first +green canonical gate. Final test-only constructor assertions were subsequently +verified in `canonical-final.log`, without changing or reinstalling the main +production artifact. Client JVM reports confirm the repository-local temporary +directory; the command relies on existing POM fork options and does not add a +separate temporary-directory option to `argLine`. + +Server selectors, JVM flags, temporary paths and seeded commands are recorded +in the local [server run manifest](/home/jara/devel/oss/questdb/core/target/schema-binding.2.6-server/run-manifest.md). +The installed standalone client was used, never `-Plocal-client`; root verified +the test classpath, final seed/temporary path, and before/after artifact hashes. +The workspace and installed main JAR share SHA-256: +`f14a8793c882bcbf5515c911376c823d8addd17e7013ed9d6f35858173b3b3d0`. +Both contain `QwpSchemaBinding` and no `QwpSchemaRowBuffer` class. + +HEADs remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with ongoing uncommitted task changes. +Both repositories pass `git diff --check`. Only the three scoped server test +files changed in this increment; no server production code, commits or pushes. +Functional tests do not establish a performance improvement or zero overhead. + +Next dependency: a real first-use DESCRIBE/feedback handoff and latest-schema +cache, with connection invalidation, bounded waits and producer-only adoption. +Do not add a feedback-only legacy probe or enable a subset of ordinary Sender +setters. The remaining conversion/inference contract and owner integration must +be coherent before automatic schema mode is activated. + +## D018 — Iteration 2.7: one lookup handoff and one latest-schema cache + +Date: 2026-09-10. Status: accepted; final verification below. + +The next functional slice connects existing schema codecs and ACK/NACK feedback +to the real cursor I/O loop. Root owns architecture, decisions and acceptance; +SOL owns client implementation, real-server tests and independent/adversarial +review. Ordinary Sender activation and row-boundary adoption remain out of +scope. This must produce an observable real-loop E2E, not another unused codec +or a feedback-only legacy probe. + +Decisions and source-checked simplifications: + +- One producer needs one outstanding lookup, not a general request queue. A + coordinator monitor protects the request and a lazy insertion-order cache + capped at 1,000,000 entries. Do not combine a concurrent map, eviction queue, + atomic request slot and separate state machine without a demonstrated need. +- Low-level `CursorWebSocketSendLoop.resolveSchema(name, timeoutMillis)` uses + the cache or issues DESCRIBE; `refreshSchema` forces discovery for subsequent + conversion revalidation. Return known or confirmed-missing immutable + responses. Existing typed schema errors distinguish access denial, + unavailable lookup and unsupported capability/size. Preserve interruption. + Neither method changes a row, binding, buffer, SF log or ACK watermark. +- The deadline includes queueing, connection wait, control send and response. + Loss releases the current waiter; a later request can wait for reconnect + within its own deadline. Installing a replacement connection clears latest + entries, not pinned snapshots or a newer unsent request. Close and terminal + exit release waiters before waiting for worker shutdown. IDs are never + reused; late or unmatched replies are discarded, not opportunistically cached. +- Cache known and confirmed missing results. Do not cache denial/unavailability; + forced refresh evicts the previous entry on admission instead of duplicating + cleanup across every failure branch. A failure never restores that entry; + independently received newer feedback remains usable. Preserve known entries + during an outage for offline row construction. Use QuestDB-compatible character folding, not + locale-sensitive whole-string lowercasing. +- A proposed SCHEMA/durable-ACK discriminator collision was disproved before + implementation: `KIND_SCHEMA=2` is inside the QWP control envelope, whereas + durable ACK starts with status 2. The full control message starts with QWP + magic. Strict envelope routing needs no protocol change or decoder fallback. +- Root and both reviewing SOL passes verified server response ordering: + `QwpIngressUpgradeProcessor.handleSchemaControl` sends its sampled response + before pending ACKs; blocked responses finish before later response handling; + `trySendAck` and `sendErrorResponse` sample current metadata when serializing. + Pending feedback retains names rather than stale snapshots. Therefore use + receive order; a client wire-sequence barrier would incorrectly suppress + newer metadata attached to an ACK for older data. No per-table revision + comparison or new server ordering field is needed. +- The existing WebSocket send path already handles whole-frame partial writes. + Send one control request at a loop boundary with its remaining deadline; + recycle after a transport/partial-send failure before sending another frame. + Do not introduce a second socket owner or send-time data transformation. + +Validation brief: public loop/network tests for cold/hit/refresh, case aliases, +missing versus failures, feedback/invalidation ordering, control/durable ACK +interleaving, timeout/cancellation/late replies, connection and close lifecycle, +and unchanged data sequence/watermarks. Real-server E2E must obtain metadata +through this loop, publish bound data, alter INT to LONG, receive updated cache +metadata through ACK, clear/rebind the same buffer, and verify exact wire +identities and SQL values. No reflection, private-field inspection, test-only +production hooks or fixed sleeps. Client/server Maven runs remain serialized +per repository, with repository-local temporary paths and installed standalone +client artifact provenance; never use the server local-client profile. + +Performance boundary: the hardware-first skill informs bounded I/O work and +allocation/ownership scrutiny. This is functional integration, not a measured +optimization. No isolated schema-coordination workload has yet established +throughput, latency or allocation cost; do not claim a speedup or zero overhead. +Keep the benchmark study separate from this bounded implementation. + +### Iteration 2.7 implementation and review checkpoint — 2026-09-10 + +The client implementation adds package-private `QwpSchemaCoordinator`, the two +documented low-level loop methods, and a byte-array WebSocket send overload +that reuses the existing whole-frame send path. No ordinary Sender setter, +row/buffer ownership or SF engine change was needed. The coordinator retains +one request object and its sending-client identity; there is no separate epoch, +atomic request state machine, callback adapter, subscription or sequence barrier. + +Root and independent SOL review corrected draft lifecycle races before final +validation: closed coordinators must reject cache hits and later feedback; +connection loss must be reported once through the common reconnect boundary; +pre-completing a failed send could otherwise cancel a subsequent request twice. +Close releases waiters before joining the I/O worker. The control-send path +propagates JVM errors to the existing fatal-exit handling. Deadline arithmetic +uses elapsed nanoseconds rather than assuming `nanoTime()` has a positive +origin. Request validation/ID assignment happens before publication, and table +name length is bounded before normalization. + +Cache storage is lazy. Invalidation drops the map reference rather than +walking up to a million entries while holding the monitor. Feedback is decoded +completely before publication, with bounded per-name monitor work; a close +guard prevents late publication. Ordinary responses without metadata bypass +the coordinator, and an idle loop checks a volatile pending flag without +taking the coordinator monitor. These are source-level ownership/work bounds, +not measured throughput, latency or allocation results. + +Independent SOL source/adversarial review accepted the final production code +and test refinements with no remaining reported blockers. Root additionally +required explicit failed-refresh eviction coverage and a genuinely negotiated +durable-ACK test, not merely injection of a stray durable ACK into ordinary mode. +The real-server test uses existing one-byte send fragmentation, real cursor +engine/loop APIs, exact INT/LONG wire identities, ACK-derived cache lookup, +same-buffer clear/rebind and SQL values `42`, `43`, `2147483648`. + +Client gates verified by root: + +- Final focused coordinator/lifecycle suite: **12 tests**, zero failures/errors/ + skips. Log: `core/target/schema-coordination-2.7/logs/focused-post-coverage.log`. +- Final canonical gate: **352 selected tests plus two packaging checks**, zero + failures/errors/skips. Log: + `core/target/schema-coordination-2.7/logs/canonical-final.log`. +- The earlier 351-test canonical gate and installation succeeded before the + final test-only refinements. The main artifact was not rewritten while the + server tests ran. Workspace and installed standalone client JAR SHA-256: + `20308e3deb22bbf2a66355afb8fcc9529022cb8b4499c4e5730ee8b2c0fe5d8f`. + +The initial client compile failed on incorrect existing getter names; the first +combined run also exposed an invalid never-started-loop test setup (null client +without a reconnect factory). A later test compile had incorrect FSN getter +names. These were corrected, their logs retained, and they are not acceptance +evidence. No reflection, private-field inspection, test-only production hook +or fixed sleep was added. Constructor/close failure cleanup in the real-server +fixture was corrected to use try-with-resources. + +The client canonical command is the iteration 2.6 selector plus +`CursorWebSocketSchemaCoordinatorTest,CursorWebSocketSchemaLifecycleTest`, with +`-Djava.io.tmpdir=/home/jara/devel/oss/java-questdb-client/core/target/schema-coordination-2.7/tmp`. +It uses the same `mvn -B -pl core ... -Dmaven.javadoc.skip=true verify` lifecycle. +Focused runs select those two new classes with `test`; installation uses +`mvn -B -pl core -DskipTests -Dmaven.javadoc.skip=true install`. +Surefire confirms the absolute project-local temporary directory. + +The full one-million-entry eviction/resource stress fixture was **not run**. +The fixed ceiling and eldest-entry removal are source-reviewed; runtime memory +and latency at that scale remain unmeasured. This does not weaken the configured +ceiling or justify adding a test-only production capacity switch. + +Server focused/aggregate/repeat acceptance remains pending at this checkpoint. + +### Iteration 2.7 final acceptance — 2026-09-10 + +Accepted by root after independent SOL source/adversarial review, final test +review and direct verification of the client/server logs and artifact hashes. +This accepts the lookup/cache coordination slice, not ordinary Sender activation +or the complete schema-directed encoding feature. + +- Client: **352 selected tests plus two packaging checks**, all green; the + **12** new coordinator/lifecycle tests are included in 352. Final evidence: + `core/target/schema-coordination-2.7/logs/canonical-final.log`. +- Server fragmented-response focused gate: **8 tests**, all green. Evidence: + `core/target/schema-coordination.2.7-server/focused-fragmented.log` in the + server repository. An earlier unfragmented focused run also passed eight + tests; the fragmented run supersedes it for acceptance. +- Server canonical gate: **281 tests**, all green. Evidence in the same server + directory: `canonical281.log`. +- Three server feedback/conversion repeats: **20 tests each**, seeds `1/2`, + `42/43`, `1234567/7654321`, all green. Evidence in the same directory: + `repeat-1-2.log`, `repeat-42-43.log`, `repeat-1234567-7654321.log`. + +All accepted runs have zero failures, errors and skips. Exact server selectors, +JVM arguments and run-specific temporary paths are recorded in the +[server run manifest](/home/jara/devel/oss/questdb/core/target/schema-coordination.2.7-server/run-manifest.md). +Root verified the final Surefire classpath uses the installed standalone +client, the final seed pair and the absolute repository-local temporary path. +The installed and workspace JAR hashes remained +`20308e3deb22bbf2a66355afb8fcc9529022cb8b4499c4e5730ee8b2c0fe5d8f` +before and after server validation. The server local-client profile was not +used. Main production bytes were unchanged by the final test-only refinements. + +Both worktrees pass `git diff --check`. HEADs remain client +`981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`. Existing unrelated changes were +preserved. No server production code changed in this iteration; only the +existing evolution E2E was extended there. No commits or pushes were made. + +Reflection: the existing server serialization guarantee removed the need for +client ordering barriers, and actual connection identity removed the need for +an epoch counter. The coordinator fits the existing I/O owner; no second +transport, generic queue or owning row facade was added. Subsequent slices +should reuse this path for remaining conversions and producer-owned +row-boundary adoption, not add another lookup/cache layer. Full million-entry +stress and performance characterization remain explicitly unmeasured, as noted +above. Do not activate a subset of ordinary Sender setters or advertise the +full extension before the conversion and inference contract is coherent. + +## D019 — Iteration 2.7.1: reject stale replies before schema parsing + +Date: 2026-09-10. Status: accepted after client/server validation and independent review. + +A fresh SOL review after D018 acceptance identified an uncovered case: the I/O +handler decoded the entire schema body before correlating its request ID. A +malformed but identifiable reply to an expired lookup could therefore recycle +the connection and fail a newer lookup. The existing late-reply test supplied +only valid metadata. The earlier green gates did not establish this case. + +The user approved the smallest correction using existing request state. Root +owns the brief, design/journal and acceptance; SOL owns implementation and +regression tests, a separate SOL pass owns independent review, and the server +validation owner reruns existing real E2Es against the newly installed client. + +Decisions: + +- Factor fixed control-envelope validation into a small protocol helper that + safely returns the positive request ID without parsing the schema body. + Reuse validation in the full decoder rather than maintaining divergent + header parsers. Header, kind and ID occupy 21 bytes; this can identify an + absent-result reply, but the full schema decoder must still reject it. + Unsafe/unidentifiable framing retains the existing connection-failure path. +- Use one synchronized, side-effect-free predicate over the existing request, + sending-client identity, ID and deadline. Check before body decoding and + again if decoding fails. Keep decoding outside the monitor so timeout and + close are not delayed by that work. Keep the existing successful-response + check before metadata publication. +- The final positive predicate check accepts a malformed reply as a live + protocol failure. Cancellation before that check wins and the stale error + is ignored; cancellation afterward does not retroactively undo the accepted + protocol failure. No claimed-request state, epoch or additional counter is + needed. +- Reject the proposed alternative of pre-completing the request before + calling `fail()`: it would wake caller A, allow request B to queue, and then + let the common connection-loss handler cancel B too. The existing timeout + and connection-loss paths retain sole ownership of their notifications. +- No new classes, queues, callbacks, protocol version or capability changes. + Ordinary Sender activation, schema conversion, cache policy and SF/ACK + semantics remain outside this fix. + +Validation starts with observable regressions against the unfixed code, with +the failing log retained. Cover malformed identifiable expired replies both +without a pending lookup and while a newer lookup waits; the connection and +new lookup must remain usable. Current malformed bodies and unidentifiable +headers must still fail. Add fixed-header/ID boundary checks while preserving +full-decoder truncation and feedback tests. Use real scripted-peer traffic, +bounded waits and observable results, not reflection, test-only production +hooks or artificial parser delays. The precise timeout-during-decoding race +also requires source review of the post-failure predicate; do not invent a +deterministic runtime claim if that scheduling point is not exercised. + +The hardware-first skill was read for this I/O-sensitive change. Its bottleneck +study is not a fit for this narrowly authorized correctness repair; no +production optimization is attempted. The functional fallback is unchanged +ownership, bounded header inspection, parsing outside the monitor and no +performance or allocation-improvement claim. No benchmark study or broader +performance work is added to this fix. + +### Iteration 2.7.1 client acceptance — 2026-09-10 + +Root and independent SOL review accepted the production fix and strengthened +regressions. Server validation remains pending at this checkpoint. + +The initial behavioral RED ran after the unused header helper was factored in, +but before changing handler/coordinator behavior. Exactly the two new +stale-malformed-reply tests failed because the old handler decoded their bodies +and triggered connection loss. The matching-active malformed case still passed. +Evidence: `core/target/schema-reply-fix-2.7.1/logs/initial-red.log`. + +Review strengthened two tests before final acceptance: + +- Sending a stale frame immediately before a new lookup did not prove the I/O + thread consumed it with no pending request. A following schema-feedback frame + on the same connection now provides an observable barrier: zero-time cache + probes cannot send a lookup, and observing the feedback with request count + still one establishes receive order. Only then does the next lookup begin. +- A schema-unavailable error alone could have come from the lookup deadline. + The matching-malformed test now uses an effectively unbounded lookup deadline + and separately waits, within a bounded test deadline, for the public transport + error. This also respects the ordering between waiter release and terminal + error publication. New expiry tests allow one second for request delivery + instead of depending on the I/O thread running within 25 ms. + +Final client gates, all with zero failures, errors and skips: + +- Focused codec/lifecycle: **20 tests**, `focused-strengthened.log`. +- Coordinator/lifecycle repeats: **15 tests each**, `repeat-1.log`, + `repeat-2.log`, `repeat-3.log`. +- Canonical: **356 selected tests plus two packaging checks**, + `canonical-strengthened-final.log`. Earlier `canonical.log` and + `canonical-final.log` predate the final test strengthening and are superseded. +- Installation: `install.log`. + +All logs are under `core/target/schema-reply-fix-2.7.1/logs/`. Runs were serialized +and used the absolute client-repository-local `core/target/schema-reply-fix-2.7.1/tmp` +directory, verified in Surefire XML. The canonical command retains D018's +selector and `verify` lifecycle. No test classes were removed from the gate. + +The workspace and installed standalone client JARs both hash to +`b5b2f5300c7d391dae99b7f9904848c13cc77d8c22c4c4d9f11f06d86de06f94`. +The installed artifact is frozen during server validation. Production changes +are limited to the protocol helper, the existing coordinator predicate and the +I/O response handler; only protocol/lifecycle tests changed alongside them. +The precise cancellation-during-decoding interleaving remains source-reviewed, +not deterministically forced by a runtime test. No new state or test hooks were +needed. Both worktrees pass `git diff --check`; no commits or pushes were made. + +### Iteration 2.7.1 final acceptance — 2026-09-10 + +Accepted after root independently verified the final client/server logs, +standalone artifact hashes and Surefire runtime configuration, alongside SOL's +independent source and adversarial test review. + +- Server focused discovery, authorization, limits, feedback and evolution: + **19 tests**, all green, `logs/focused.log`. +- Server canonical: **281 tests**, all green, `logs/canonical281.log`. +- Server feedback/conversion repeats: **20 tests each**, seeds `1/2`, `42/43`, + `1234567/7654321`, all green, `logs/repeat-1-2.log`, + `logs/repeat-42-43.log`, `logs/repeat-1234567-7654321.log`. + +These server logs live under +`/home/jara/devel/oss/questdb/core/target/schema-reply-fix.2.7.1-server/`. +Exact selectors and JVM arguments are recorded in the +[server run manifest](/home/jara/devel/oss/questdb/core/target/schema-reply-fix.2.7.1-server/run-manifest.md). +All accepted runs have zero failures, errors and skips. The final Surefire +classpath uses the installed standalone client, not the local-client profile; +its final seed pair and absolute repository-local temporary path were checked. +The workspace and installed client JARs retain the `b5b2f530...` hash recorded +above before and after server validation. + +No server source changes were needed. Both worktrees pass `git diff --check`. +HEADs remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`; unrelated changes were preserved. +No commits or pushes were made. + +Reflection: the missing step was checking existing request identity before +interpreting a reply, not adding lifecycle state. Keeping that check +side-effect-free also avoids premature waiter notification. Most review-driven +changes were to make test ordering and failure assertions conclusive. This +accepts only the stale-reply repair; ordinary Sender activation and remaining +conversion/row-boundary work remain outside this iteration. + +## D020 — Iteration 2.8: typed microsecond/nanosecond conversion + +Date: 2026-09-10. Status: accepted after independent review and client/server validation. + +The user approved proceeding with timestamp conversion. Root recommended a +bounded conversion slice, with real-server tests from the beginning. Independent +SOL client, server and adversarial reconnaissance narrowed the slice to typed +wire-unit conversion; the broader timestamp family remains unfinished. + +Frozen scope: + +- Add `QwpSchemaBinding.timestampColumn(name, long, ChronoUnit)` for MICROS and + NANOS inputs into known, non-designated TIMESTAMP_MICRO/TIMESTAMP_NANO columns. + Encode the target's timestamp wire type and converted value directly. +- Reuse the existing timestamp drivers after the two-unit support check. No new + converter framework, binding lifecycle, buffer owner, lookup state or protocol + change. Caller-owned row cancellation and column rollback remain unchanged. +- Nanos-to-micros uses Java division by 1000, truncating toward zero, not floor. + Micros-to-nanos rejects values outside the server's exact multiplication + bounds. Same-unit conversion preserves the supplied bits. +- Timestamp cursor nullness is bitmap-only. Do not copy the separate LONG-input + source-null rule: supplied `Long.MIN_VALUE` is rejected for micro-to-nano, + becomes `-9223372036854775` for nano-to-micro, and reads as SQL NULL when + stored unchanged. Explicit nulls through the existing null-input path and + omissions use the target null representation. Verify bitmap/no-bitmap cases. +- Preserve first-value-wins before input-unit validation or conversion. + Unsupported non-null units report `UNSUPPORTED_FEATURE`; a null unit and + arithmetic overflow report `INVALID_VALUE`. These errors are not flush retries. +- Reject designated timestamp targets at the shared target-support boundary, + including the null-input path. Their bounds and server-assigned-time semantics + belong with producer-owned row completion, not this ordinary-column slice. +- Defer other ChronoUnits, `Instant`, DATE, text parsing, LONG-to-timestamp, + `at`/`atNow`, automatic row-boundary adoption and ordinary Sender activation. + This is a component extension, not complete timestamp support or a release. + +Source authority is QWP ingress, not SQL casts: + +- Server `QwpWalAppender` routes timestamp precision mismatches to + `WalColumnarRowAppender.putTimestampColumnWithConversion`; matching precision + uses the existing direct-copy path. +- The conversion method checks multiplication bounds and divides toward zero. + `QwpTimestampColumnCursor.advanceRow` obtains nullness only from the bitmap. + The initial server reconnaissance mistakenly treated the primitive minimum as + source-null; root cross-check and independent review corrected that before + implementation. +- Existing `QwpWebSocketSender.toMicros` multiplies wider units unchecked; + its `Instant` overload truncates to micros. Reusing the more permissive + timestamp-driver API without a scope gate would silently tighten overflow + handling or retain extra precision. Those changes require a separate + compatibility decision. Timestamp-to-DATE is not a supported server wire pair. + +Validation brief: shared exact-value TSV vectors across client and server, +covering all four unit/target pairs, negative fractional cutoffs, multiplication +endpoints and adjacent failures, MIN/MAX, explicit/omitted nulls and unrelated +bitmap rows. Assert target wire types, exact values and SQL stored epoch values; +compare with legacy timestamp-wire ingestion rather than SQL conversion rules. +Add A/invalid-B/C rollback, removal of failed-row-only columns, cross-setter +duplicates, unsupported/name/parameter/designated failures and buffer lifecycle +tests. Exercise plain/compressed timestamps where the existing fixtures permit. +Keep existing conversion, lookup, negotiation and replay regression gates green; +extend persisted target-byte coverage without adding another transport owner. +No reflection, private-state tests, test-only production hooks or fixed sleeps. + +Root owns the design, journal, scope and acceptance. SOL owns client code/tests +and serialized client Maven; a second SOL owns server tests and serialized server +Maven; a third independently reviews source, arithmetic and observable coverage. +The client owns the shared corpus. Coordinate installation so server tests use +one frozen standalone Maven JAR, never the server's local-client profile. +Task evidence directories are client `core/target/schema-timestamps-2.8/` and +server `core/target/schema-timestamps.2.8-server/`, with absolute local JVM temps. + +Baseline client HEAD is `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD is +`12a33d651e51e2682e7a448c8db5168fc72dfad3`. Workspace/installed standalone client +JARs both hash to +`b5b2f5300c7d391dae99b7f9904848c13cc77d8c22c4c4d9f11f06d86de06f94`. +Both repositories contain prior accepted, uncommitted work. Preserve it and the +unrelated server Rust work; no commits or pushes are authorized by this slice. + +The hardware-first skill was read because this extends the write path. Its +bottleneck/optimization gate is not a fit for a functional conversion extension; +use the established functional fallback, with no production optimization or +performance/zero-allocation claim. Reuse existing buffer ownership and conversion +utilities. Hardware cost and scale measurements remain unmeasured, not implied +by passing functional tests. + +### Iteration 2.8 client acceptance and early real E2E — 2026-09-10 + +Root and independent SOL review accepted the bounded client implementation. +`timestampColumn` reuses the two existing drivers, writes the exact target type +and adds no state. Full-type matching preserves nano precision bits; unsupported +targets and duplicate suppression precede arithmetic. The shared designated +guard also rejects the existing null-input path. Caller-owned row lifecycle, +ordinary Sender, protocol, cache and replay implementation are unchanged. + +The shared corpus grew to **48 vectors**. Root and the independent reviewer each +checked it using exact BigInt arithmetic rather than production conversion +utilities: zero mismatches. Client/server resource mirrors are byte-identical, +SHA-256 `696d832cd692d5ff8765ac265070a8af6fcfcdc8c4cc87dc5967d72f4d946ab6`. + +Client gates have zero failures, errors and skips: + +- Focused binding: **26 tests**, `logs/focused-final.log`. +- Final canonical: **359 selected tests plus two packaging checks**, + `logs/canonical-final.log`, with the final 48-vector resource. +- Installation: `logs/install.log`; workspace and installed standalone JARs + hash to `909f1e039f6528067545b56477bdb8beab829a7c37f8151e9fb6ddfe6ff960ce`. + The final client verify ran without reinstalling while server validation used + the frozen installed artifact; both hashes remained equal afterward. + +Client logs are under `core/target/schema-timestamps-2.8/`. The canonical selector +is D019's unchanged selector: the three new tests live in `QwpSchemaBindingTest`. +Surefire confirms the absolute client-repository-local task temporary directory. +The first two focused runs failed because new test readers omitted timestamp +encoding/bitmap bytes and expected a never-bound column in the sparse layout. +These were fixture mistakes, not product failures or a claimed behavioral RED; +their logs remain as development evidence. The existing layout/encoder was not +changed to satisfy them. + +The first real-server timestamp E2E passed **3 tests** in +`core/target/schema-timestamps.2.8-server/logs/focused.log`. The expanded source +then passed **6 tests** in `logs/focused-final.log`, adding schema-bound MIN and +bitmap cases, actual Gorilla encoding and recovered timestamp delivery. + +Root review requested exact per-row wire and SQL checks for the compressed case, +using nanos input with a nonzero sub-micro remainder. Count/min/max alone could +hide incorrect middle values. The SF smoke proves exact target encoding before +persistence and recovered delivery with the expected value, not byte-identical +replay by itself. Existing generic replay tests separately check byte identity; +the independent review's stronger wording was corrected. Final server gates +will run after these small test refinements. No server production change is +needed, and no performance claim follows from these functional results. + +### Iteration 2.8 final acceptance — 2026-09-10 + +Accepted by root after independent SOL source/adversarial review and direct +verification of final logs, corpus equality, artifact hashes and runtime +configuration. All accepted runs have zero failures, errors and skips: + +- Client canonical: **359 selected tests plus two packaging checks**, + `core/target/schema-timestamps-2.8/logs/canonical-final.log`. +- Server refined timestamp E2E: **6 tests**, `logs/focused-final2.log`. + This supersedes the earlier three-test and six-test focused runs for final + test-source acceptance. +- Server canonical: **287 tests**, `logs/canonical287.log`. +- Server conversion/feedback repeats: **26 tests each**, seeds `1/2`, `42/43`, + `1234567/7654321`, logs `repeat-1-2.log`, `repeat-42-43.log` and + `repeat-1234567-7654321.log` under `logs/`. + +Server evidence lives under +`/home/jara/devel/oss/questdb/core/target/schema-timestamps.2.8-server/`. +Exact selectors, JVM flags and temporary directories are recorded in the +[server run manifest](/home/jara/devel/oss/questdb/core/target/schema-timestamps.2.8-server/run-manifest.md). +The final Surefire report identifies the installed standalone client, final +seed pair and absolute task-local JVM temp path. No local-client profile was +used. The workspace and installed JARs retained the full `909f1e03...` hash +recorded above throughout server validation, and the two 48-vector corpus +files remained byte-identical. + +The production change is limited to `QwpSchemaBinding`: one timestamp input +method, existing driver reuse, exact target wire mapping and the shared +designated-target support guard. The client binding tests, shared timestamp +corpus and new server E2E test provide the additional coverage. No server +production change, new converter abstraction, state or buffer owner was needed. +Both worktrees pass `git diff --check`. HEADs remain the baseline revisions +recorded above; prior/unrelated work is preserved. No commits or pushes were made. + +Reflection: typed wire-unit conversion is a useful complete subproblem. It +does not justify claiming all timestamp overloads work. Source-backed review +prevented an unchecked assumption about timestamp source nulls and prevented +existing driver reuse from silently redefining larger-unit/Instant behavior. +Wire and SQL checks establish conversion before ingestion; exact per-row +compressed checks and recovered-delivery coverage supplement that contract. +The next timestamp work needs an explicit overflow/precision decision for +broader input normalization. Designated timestamp handling remains tied to +producer-owned row completion, and ordinary Sender activation remains deferred +until the full conversion/inference contract is coherent. + +## D021 — Decide broader timestamp input semantics before implementation + +Date: 2026-09-10. Status: contract approved by the user; iteration 2.9 accepted. + +The user asked to continue after D020. Root inspected the next timestamp slice +and requested independent SOL compatibility review. The design explicitly +requires an overflow/precision decision before implementing broader units and +`Instant`; continuing implementation does not silently resolve that product +choice. No production code or tests changed in this step. + +Independent SOL review agrees with the recommendation and the need for explicit +approval. It also confirmed the intermediate-overflow issue in both timestamp +drivers and the need to preserve the existing unit allowlist. + +Current source establishes two differences from direct target conversion: + +- `QwpWebSocketSender.toMicros` multiplies MILLIS through DAYS unchecked. + For example, `Long.MAX_VALUE` seconds wraps to `-1000000` microseconds instead + of reporting overflow. Root verified the arithmetic independently with BigInt. +- Its `Instant` overload first converts to micros, even for a server nano + column. An instant 999 nanoseconds after the epoch consequently becomes zero + on that path; direct nano conversion would retain 999. + +Root recommends the following contract for schema mode only, subject to user +approval: + +- Reject inputs outside the target timestamp's range with `INVALID_VALUE` + before append, rather than preserving accidental arithmetic wraparound. +- Preserve all `Instant` precision supported by the target: retain nanos for + TIMESTAMP_NS, and retain existing `Instant`-to-micro rounding for TIMESTAMP. + Do not change the already-approved primitive NANOS-to-MICROS truncation rule. +- Support the existing QWP unit set: NANOS, MICROS, MILLIS, SECONDS, MINUTES, + HOURS and DAYS. Do not accidentally admit additional units through the + timestamp drivers' generic Duration fallback. +- Preserve duplicate suppression before conversion/argument validation. A null + `Instant` reports `INVALID_VALUE`, not an implicit timestamp null. +- Keep old-server/legacy behavior, designated timestamp handling and ordinary + Sender activation outside this slice. The changed overflow acceptance and + added nano precision must be documented as intentional schema-mode behavior. + +Implementation warning discovered before delegation: the current timestamp +drivers multiply epoch seconds with `Math.multiplyExact` before adding the +positive fractional part. Near the negative limit that intermediate product +can overflow even when the final timestamp fits. For example, epoch seconds +`-9223372037` and nano fraction `145224193` represent `Long.MIN_VALUE + 1` +nanoseconds. Root's BigInt check confirms the final value is representable; +blindly reusing `NanosTimestampDriver.from(Instant)` would reject it. If this +contract is approved, require range-correct arithmetic and boundary tests for +the final combined value, without introducing BigInteger into the write path. + +The next step is one user decision on the recommended schema-mode overflow and +precision contract. The accepted 2.8 artifact and all existing behavior remain +unchanged while that decision is pending. + +### Approval and iteration 2.9 scope + +The user approved the recommendation with "ok". Design revision 22 now records +the strict target-range and full-nanosecond `Instant` contract. The pending +decision language above records the earlier pause, not the current status. + +Implement only broader units and `Instant` on the existing non-owning binding. +Reuse the checked primitive drivers after an explicit unit gate, but keep the +range-correct `Instant` arithmetic local to schema conversion. Do not change the +shared drivers, legacy Sender, designated timestamp handling, protocol or row +ownership. This avoids broadening a schema-only decision into unrelated APIs. + +SOL assignments: client implementation/component tests and serialized client +builds; separate real-server E2E/shared vectors and serialized server builds; +independent arithmetic, compatibility and adversarial review. Root owns design, +integration and acceptance. Preserve the existing 2.8 corpus and add explicit +new input vectors, wire assertions, SQL boundary/precision checks, local-error +rollback, legacy comparisons and an `Instant` recovered-delivery case. + +The hardware-first performance skill was consulted because this changes a +write path. This is a functional correctness increment, not a bottleneck or +optimization study; its hardware-measurement gate cannot establish this feature's +contract. Keep arithmetic bounded and avoid new per-write state or BigInteger. +Hardware limits, throughput and allocation rates remain unmeasured; make no +performance improvement or zero-allocation claims. Production optimization is +outside this slice. + +### Simplification and pre-test review + +The author initially proposed checking floor-divided long bounds and handling +the lowest representable second separately. Root recommended a smaller formula: +for negative seconds, evaluate `(seconds + 1) * scale + (fraction - scale)` +with checked multiplication/addition; otherwise evaluate `seconds * scale + +fraction` with the same checks. The normalized fraction is in `[0, scale)`, so +both terms in the negative branch are non-positive. An intermediate underflow +therefore cannot hide a representable final result. No special boundary table +or new converter state is needed. + +SOL independently checked the argument and boundary arithmetic. Root also +compared checked arithmetic with an exact BigInt oracle over 200,140 boundary +and deterministic sampled cases at both scales (seed 20260910): zero mismatches. +This is an independent arithmetic check, not a substitute for component or SQL +tests. Initial independent production review found no correctness or scope issue; +test and final acceptance review remain pending. + +### Implemented behavior and validation + +The production change is confined to `QwpSchemaBinding`: extend the primitive +unit gate and add the `Instant` overload plus one small arithmetic helper. No +new schema state, buffer owner, protocol field or send-time work was introduced. +The common timestamp drivers and `QwpWebSocketSender` are byte-for-byte unchanged +from this iteration's starting source hashes. The 48-row 2.8 unit corpus is also +unchanged. + +The new shared `timestamp-inputs.tsv` has 102 cases: zero, signed values and both +range boundaries for each newly supported unit/target pair; primitive minimum +longs; `Instant` extrema, negative fractions, target precision and exact long +boundaries. Root added two precision-edge requirements during review: the extra +999 nanos within the boundary microsecond must not cause rejection. The minimum +microsecond remains a wire-present minimum long/SQL NULL; the maximum remains a +finite maximum long. Independent SOL proleptic-Gregorian/BigInt verification of +all final vectors found zero mismatches. + +Component tests assert target wire types/values, typed errors, duplicate-before- +validation ordering, partial-row and failed-column rollback, designated-target +rejection and stale-binding rejection through the new overload. The real-server +corpus test checks each accepted encoded value before sending and the resulting +SQL value; rejected inputs fail locally. The ordinary Sender test deliberately +pins the old results: maximum-long seconds wrap to -1000000000 in a nano column, +and an `Instant` 999 nanos after the epoch becomes zero. Schema conversion does +not reproduce those legacy normalization rules. + +The existing SF test now also encodes an `Instant` as exactly 123 nanos before +append, closes the cursor engine, then drains through a recovering Sender and +checks SQL. It proves pre-persistence conversion and recovered value preservation, +not exact replayed-byte identity by itself. Existing generic replay tests retain +that separate byte-identity coverage. + +Validation used serialized Maven within each repository, an installed standalone +client JAR (never `-Plocal-client`) and absolute project-local JVM temp directories. +Root verified final log results, matching corpora, stable main JAR hashes and +unchanged legacy/common-driver source hashes. All accepted runs have zero +failures, errors or skips: + +- Client focused: 89 tests with the final corpus; 29 binding tests after the + final stale-binding assertion. +- Client canonical after final review: 362 selected tests plus 2 packaging checks. +- Server focused: 8 timestamp E2E tests, including the existing six scenarios. +- Server canonical: 289 selected tests. +- Server seeded repeats: 28 tests each with seeds 1/2, 42/43 and + 1234567/7654321. +- Both repositories: `git diff --check` clean. + +Exact commands, selectors, runtime identities and log paths: + +- Client: `core/target/schema-timestamps-2.9/run-manifest.md`; + final broad log `logs/canonical-post-review-final.log`. +- Server: sibling `questdb/core/target/schema-timestamps.2.9-server/run-manifest.md`; + final broad log `logs/canonical289.log` and three `logs/repeat-*.log` files. +- Client HEAD: `981bdb02a471f3b290c89b8e78cbc422610e329e`; + server HEAD: `12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with the existing + schema worktree changes. JDK: Corretto 25.0.4; Maven: 3.9.11. +- Workspace/installed main JAR SHA-256: + `5ae6591c5ba90a29f4b743bfba0259488b4e16c0c76f59a09b29def745de08e2`. +- Both copies of the new corpus SHA-256: + `f46a23969089a41ea38bf1d801f0176e8c2fd9fef9a91c71f471a64deae169df`. + +An initial focused client run exposed an obsolete 2.8 test expectation that +MILLIS was unsupported; it now tests unsupported WEEKS. This was a fixture update +for the approved scope, not a production regression. A server run was interrupted +during compilation while the final corpus additions were synchronized; its +replacement completed successfully. Neither superseded run is counted as an +acceptance gate. The main JAR was not reinstalled after test-only additions; +server tests use its unchanged production bytes and their final mirrored corpus. + +Final independent source/test review accepted with no findings after the corpus +and new overload guards were synchronized. A second SOL adversarial review of +the client implementation also found no issue. The client author independently +cross-reviewed the server assertions and found no false-green gap against the +stated scope. That review confirmed the limited SF claim above and distinguished +the two explicit legacy-output tests from exhaustive legacy equivalence; the +broader unchanged-code claim also rests on the source-hash checks. + +Root accepts iteration 2.9. No commits or pushes were made. This remains a +conversion component, not activated schema mode in ordinary Sender. Timestamp +text parsing, designated timestamp row completion, remaining conversion families, +inference, automatic row-boundary adoption and coherent Sender activation remain +explicitly unfinished. The useful simplification is the bounded arithmetic +helper and reuse of the existing primitive drivers, not a new converter framework +or duplicated row lifecycle. + +## D022 — Validate Sender integration prerequisites before activation + +Date: 2026-09-10. Status: source-backed readiness review complete; activation deferred. + +The user explicitly required sufficient context and assumption validation before +integrating senders. Root traced the current Sender build, row completion, +rollback, buffering, encoding, publication and schema APIs against design +revision 22. Three SOL agents independently inventoried setter/server conversion +coverage, negotiation/recovery and adversarial row/schema transitions. No +production code, tests or build artifacts changed; no tests were rerun. This +journal entry records the findings under the standing decision-log agreement. + +### Confirmed gaps, not assumptions of readiness + +- **The binding is not a replacement for the Sender write surface.** Its public + inputs are LONG, STRING, UUID and timestamp overloads; non-null STRING currently + converts only to UUID. Even ordinary STRING/VARCHAR, SYMBOL, BOOLEAN and DOUBLE + identity writes are not covered. Compare `QwpSchemaBinding.java:95-243,324-373` + with `QwpWebSocketSender.java:1118-1248,1571-1738,2285-2484,2892-3017` and the + server's `QwpWalAppender.java:522-885`. Complete the agreed accepted conversion + matrix, not just method signatures or identity writes, before activation. +- **Existing setters are not interchangeable for nulls.** Decimal, array and + IPv4-text nulls are early no-ops; binary null is rejected. Preserve the explicitly + retained no-op rules. Schema-mode cross-setter duplicate suppression is an + intentional change and must run before value conversion; do not accidentally + impose all legacy mismatch/validation ordering on schema mode. +- **Inference and designated timestamps are unfinished.** The binding rejects + missing tables/columns and designated columns (`QwpSchemaBinding.java:297-332, + 407-415`). Sender currently creates its inferred layout and completes rows via + `at`/`atNow` (`QwpWebSocketSender.java:1016-1055,2924-2952,3111-3127`). Confirmed + missing metadata may use inference; unavailable metadata must never do so. + `at` needs the pinned target unit; a setter-free `atNow` still needs resolution. +- **Normal construction does not yet select schema mode.** `buildAndConnect` + requests the extension only when schema is already required + (`QwpWebSocketSender.java:3378-3379`). Existing required-capability latches protect + recovery, but are not a substitute for initial mode selection. ASYNC currently + starts with no client and marks Sender connected after starting its I/O loop + (`:4047-4064,4176`); that flag does not prove a successful handshake. The future + first effective write must wait for initial negotiation. Only a successful + old-server handshake permits legacy writes; a timeout/offline server does not. +- **A table selection is not a row boundary.** Repeating the current `table()` + returns immediately (`:2924-2929`), and another row may start without calling + `table()` again. Detect the first effective operation from actual row state, + pin once, and keep that snapshot through completion/cancellation. Clear both + designated timestamp caches when replacing a layout, not only on table switch. +- **Existing flush is publication, not just sealing.** Sender retains one current + buffer per table, encodes legacy frames (`:4335-4353`), then immediately appends + through `sealAndSwapBuffer` (`:5154-5200`). `QwpTableBuffer.reset()` preserves + column definitions and binding (`:359-367`); `clear()` destroys them and data + (`:123-136`). Neither alone retains old completed rows while adopting new types. + Preserve old encoded blocks and existing pending accounting, symbol dictionary + ordering, split-frame behavior and per-table transaction/commit boundaries. +- **Refresh is available but not integrated with row errors.** The loop exposes + resolve/refresh (`CursorWebSocketSendLoop.java:1226-1239`); Sender calls neither. + `SCHEMA_CHANGED` currently appears only in the reason enum. Existing evolution + E2Es explicitly reset/clear/rebind their component buffers + (`QwpSchemaFeedbackEvolutionE2ETest.java:76-134`); they do not prove automatic + Sender adoption or whole-row retry after a cached conversion rejection. + +API presence also does not prove successful server ingestion: current client +`longArray` overloads emit TYPE_LONG_ARRAY, but this server's mapping explicitly +rejects that wire type (`QwpWalAppender.java:124-125`), with a real-server rejection +test (`QwpSenderE2ETest.java:3628-3636`). Do not count it among working baseline +writes, or infer a new LONG_ARRAY conversion contract from the method name. +Validate the design's array-input contract in its own conversion slice. + +### Shortcuts challenged during review + +Root rejected two initial SOL recommendations: routing only the implemented +setters while retaining raw/server-side conversion for the rest, and permitting +ASYNC rows before initial negotiation. Both contradict revision 22. The agents +re-read the contract and corrected their recommendations. Existing legacy rows +may survive an old-to-new upgrade only after legacy mode was established by a +successful old-server connection; they are not permission for offline guessing. + +The adversarial review also corrected an overly broad rejection of sealing on +schema change. Internal sealing is allowed; calling the current publishing +flush path unconditionally is not proven safe. A retained local encoding or +deliberately deferred publication could reuse existing machinery, but each must +prove ordering, accounting, reset/close, crash/recovery and backpressure behavior. +Do not choose a new per-table generation container or queue framework merely +because multiple encoding snapshots must coexist. Likewise, reuse the existing +request flag and mandatory-capability latches before adding negotiation state; +requesting support and requiring it are distinct semantics, not justification +for another owner or user-visible switch. + +### Observable integration acceptance sequences + +1. Buffer an INT/v10 row, learn LONG/v11, and write the next row without another + `table()` call. Verify separate old/new wire types and identities plus SQL. + Repeat with feedback arriving mid-row: that whole row stays on its old schema. +2. Exercise `at`, setter-free `atNow`, no-op/null inputs, duplicate inputs and + cached timestamp references across row/layout transitions. +3. Reject against a cached schema and refresh once: unchanged target gives the + invalid-value result; changed target/incarnation cancels the whole row and + gives `SCHEMA_CHANGED`; refresh failure gives its actual reason. Earlier + completed rows survive and the next whole-row attempt can succeed. +4. With all auto-flush triggers disabled, buffer rows across two tables and a + schema change. Prove no unintended publication/commit, correct explicit-flush + results and unchanged reset/crash behavior. Repeat with transactional + auto-flush, capacity pressure, dictionary splitting and reconnect. Preserve + existing per-table guarantees; do not claim new cross-table atomicity or + stronger delivery guarantees after partial publish failures. +5. Test successful old/new handshakes, ASYNC first-write timeout, old-to-new + upgrade with pending legacy rows, reconnect during a partial row and extended + SF/background recovery. Never guess legacy, downgrade or relabel bytes. +6. Cover ordinary working setter/target pairs and confirmed missing-table/column + inference alongside unavailable/denied metadata. No raw-value fallback or + per-setter opt-out on supporting servers. + +Recommendation: continue bounded converter/inference and row-lifecycle work +without activating normal Sender. Lock down existing Sender behavior with +observable characterization tests where needed. Public activation must land with +the complete integration matrix above, not ahead of it. Reuse the existing row +owner, schema coordinator, encoder and recovery path; choose any additional +storage/state only after those sequences establish why it is necessary. + +## D023 — Text and symbol inputs reuse target-native text encoding + +Date: 2026-09-10. Status: iteration 2.10 accepted after review and validation. + +The user asked to continue after the integration-readiness review. The next +bounded slice covers six pairs on the existing non-owning binding: STRING and +SYMBOL inputs into exact STRING, VARCHAR and SYMBOL schema targets. Leave normal +Sender, inference, designated timestamps, other conversions and activation alone. +This fills ordinary text coverage without loosening the D022 activation gate. + +Root and independent SOL source review verified: + +- The server accepts all six pairs (`QwpWalAppender.java:698-805`). STRING and + VARCHAR targets share TYPE_VARCHAR on QWP; SYMBOL retains TYPE_SYMBOL. +- Existing `ColumnBuffer.addString` encodes text into its offsets/data storage; + `addSymbol` supports both the owner-backed global dictionary and a senderless + local dictionary (`QwpTableBuffer.java:1197-1228,1855-1863`). Its constructor + documentation overstates the need for a Sender, but the implementation already + provides the needed local path. No new dictionary or owner is necessary. +- Both active client serialization paths replace individual unpaired UTF-16 + surrogates with literal `?`, preserve following characters and encode valid + pairs normally (`OffHeapAppendMemory.java:166-202`, + `NativeBufferWriter.putString`/`Utf8s.strCpyUtf8`). Preserve this behavior rather + than substituting a parser/encoder or changing shared helpers. +- Null is distinct from empty. Text value case and whitespace must not be + normalized; column-name case folding is unrelated. Distinct raw strings can + occupy separate symbol IDs and still encode to the same replacement text. +- Text writes and dictionary insertion snapshot mutable inputs using existing + encoding/copy operations. Tests must mutate a supplied CharSequence after the + setter and verify the buffered value, not inspect ownership fields. + +Implementation: extend `stringColumn` and the existing exact target-to-wire map, +then add `symbol(name, value)` with the same duplicate-first buffer lookup. +Preserve STRING-to-UUID and existing broader STRING-null behavior. The new SYMBOL +input accepts null only for its implemented text targets; reject other pairs +unless the value is already suppressed as a duplicate. Keep unknown full target +types, extension parameters and designated columns behind existing guards. + +SOL owns client implementation/component tests/shared corpus, a separate real +server E2E implementation and serialized server builds, and independent source/ +corpus/adversarial review. Root owns scope, final cross-review and acceptance. +The corpus records input UTF-16 units, expected target wire type/UTF-8 bytes and +expected SQL text units explicitly. Cover all six pairs, null/empty, Unicode, +malformed surrogate sequences, controls, dictionary collisions, duplicates, +rollback, mutable inputs and reset/clear. Also test existing global dictionary +encoding and SF recovery through production APIs, with no reflection, TestOnly +accessors, new hooks or fixed sleeps. Assert encoded bytes and real SQL, not just +successful sending. Existing corpora remain unchanged. + +The hardware-first performance skill was consulted for the write-path change. +This is functional coverage, not production optimization: reuse existing storage +and serialization and add no new state/framework. No hardware bottleneck, +throughput improvement or allocation-rate claim is made. As in D021, measurement +gates are not a substitute for conversion correctness. Use absolute project-local +JVM temp storage; `/tmp` remains nearly full. Preserve all existing work and do +not commit or push. + +Review clarified an existing encoder precondition: a Sender-owned symbol buffer +contains global IDs, so it must use the global dictionary message path. The +senderless convenience encoder uses a local dictionary and cannot encode that +buffer correctly. Do not conceal this by introducing another dictionary owner or +an automatic format conversion. The owner-backed test constructs a normal Sender, +lets the binding intern new values, then uses a matching explicit test dictionary +to assert the global delta and actual row IDs through the production encoder. +Only a prefix is registered before binding writes; registering every tested value +in advance would miss the insertion behavior. Distinct raw keys with identical +replacement text must retain distinct client IDs. + +Coverage remains deliberately separated: the client fixture proves owner-backed +global encoding; real-server component and SF tests prove the existing local +dictionary path; ordinary Sender tests establish legacy value parity. Together +these do not prove activated schema-aware Sender delivery or global SF replay. +Those remain obligations of the D022 integration gate. + +### Review and test corrections + +Independent SOL review checked all 90 vectors with a separate UTF-16/UTF-8 +oracle: zero mismatches. Root strengthened the tests to assert reset output on +the wire, enforce 90 corpus records and 15 records per conversion pair, and +preserve a non-null completed A row across failed B and completed C. Server +checks also distinguish explicit null and omission from empty text using raw +values and SQL `is null`, not the text printer's blank-cell rendering. + +The broader server suite exposed an obsolete assertion in +`QwpSchemaRowBufferE2ETest`: STRING-to-STRING had deliberately been unsupported +in the earlier UUID-only slice. Preserve that test's unsupported-column rollback +coverage using LONG-to-STRING, which is still outside this binding's supported +conversions. Do not remove the assertion or turn off the regression suite. + +The directly related `QwpTableBuffer` constructor Javadoc was corrected to +describe its global/local dictionary choice. This is the sole change to that +existing buffer class in this iteration: documentation only, with unchanged line +count, made after the client gates/install. The executable artifact remains +frozen for server validation; no reinstall occurs during server tests. + +Test-authoring failures remain in the run manifests: Java-11-incompatible switch +expressions in a new helper, an incorrect close-flush configuration key, and the +SQL null-text printing expectation. These did not require production fixes. +The shared TSV intentionally has trailing delimiters on its six empty-text +records: these retain seven fields and must not be stripped as whitespace. + +### Final validation and reflection + +All final gates passed with zero failures, errors or skips: + +- Client focused: 35 tests (29 existing binding tests and 6 text tests). +- Client canonical: 368 selected tests plus 2 packaging checks. +- Server focused: 8 tests across text and the existing row-buffer class. +- Server canonical: 293 selected tests. +- Server seeded repeats: 32 tests each with seeds 1/2, 42/43 and + 1234567/7654321. + +The canonical selectors extend D021 with `QwpSchemaBindingTextTest` on the client +and `QwpSchemaTextE2ETest` on the server; the latter also joins all three seeded +repeats. Exact commands, runtime identities and superseded-run explanations are +in `core/target/schema-text-2.10/run-manifest.md` and sibling +`questdb/core/target/schema-text.2.10-server/run-manifest.md`. Final logs are +client `logs/canonical-final.log`; server `logs/focused-final3.log`, +`logs/canonical293-final.log` and the three `logs/repeat-*.log` files. + +Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD remains +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with the existing dirty schema +worktrees preserved. Runtime: Corretto 25.0.4 and Maven 3.9.11. Builds were +serialized within each repository; server runs used the frozen installed client +JAR, never `-Plocal-client`, and absolute project-local JVM temp directories. + +- Workspace/installed main JAR SHA-256: + `572065d10739860e6b63d80f55f6834429f2b9134c360519d7e9aed094d08443`. +- Both 90-record text corpus copies SHA-256: + `b5bdb1102a6d0887c8c81acc0eabf7519b56ac809f05bc505b44b92ccdbcaba7`. +- Root rechecked unchanged Sender, column writer, global dictionary and both + UTF-8 serialization source hashes, and unchanged timestamp corpora. Restoring + only the old constructor comment in a read-only stream reproduces the exact + pre-iteration `QwpTableBuffer` source hash; all executable buffer code is + unchanged. Both repositories pass `git diff --check`; untracked Java checks + are clean, with the intentional TSV delimiters noted above. + +Independent SOL source/test review accepted the final code, corpus, dictionary +fixture and corrected old test. The client author also cross-reviewed real-server +assertions; the server author independently reviewed the client conversion and +dictionary paths adversarially, finding no defect. Root checked the actual +wire/SQL assertions, logs, corpus hashes, final server classpath/temp/seed +properties and scope boundaries rather than treating review votes as acceptance +evidence. + +Root accepts iteration 2.10. The useful simplification is reusing existing text +storage, symbol dictionaries and encoding paths: six conversions need no new +production state, generic converter framework or row owner. This is a functional +result, not a measured performance claim. Normal Sender remains unmodified by +this slice, and D022's conversion, inference, negotiation and row-boundary +integration requirements remain open. No commits or pushes were made. + +## D024 — Boolean conversion uses target values and strict ASCII parsing + +Date: 2026-09-10. Status: iteration 2.11 accepted after review and validation. + +Continue with the BOOLEAN component family, not ordinary Sender integration. +Root and separate SOL source reviews confirmed a small complete scope: +`boolColumn(boolean)` into BOOLEAN, BYTE, SHORT, INT, LONG, FLOAT, DOUBLE, STRING +and VARCHAR, plus `stringColumn` into BOOLEAN. BOOLEAN-to-SYMBOL and other +unimplemented pairs remain unsupported. Preserve every existing conversion and +the broader `stringColumn(null)` rule; adding a BOOLEAN wire mapping must not +implicitly authorize other input methods. + +### Source-backed contract and implementation boundary + +- `QwpWalAppender.java:555-563,608-616,698-805` and + `WalColumnarRowAppender.java:228-330` establish the nine boolean targets. + Convert directly before append: boolean, exact numeric zero/one or lowercase + ASCII text. Do not retain BOOLEAN wire for a numeric/text target and leave + conversion to the server. +- `WalColumnarRowAppender.java:1241-1268` accepts only single-character `0`/`1` + and ASCII-case-insensitive `true`/`false`. Whitespace, signs, alternate words, + embedded NUL, Unicode lookalikes and malformed surrogate sequences are invalid. + The existing client `Chars.equalsLowerCaseAscii` uses strict ASCII folding and + can be reused without allocating strings or encoding temporary UTF-8. Java + `Boolean.parseBoolean` and Unicode case-insensitive comparison have different + acceptance rules and must not replace this contract. +- BOOLEAN storage has no distinct SQL null: the server maps wire null to false. + BYTE/SHORT similarly store zero. Nullable numeric and text targets retain their + own nulls. This follows the already approved target-native missing-value rule, + not a new policy choice. Legacy boolean layouts instead materialize omitted + values as false before server conversion, giving numeric zero or text `false`. + Test both behaviors explicitly without changing legacy Sender. +- `ColumnBuffer.addBoolean` already stores ordinary boolean values; the column + writer emits a row-null bitmap and then packs only non-null value bits. + `QwpBooleanColumnCursor` advances its value index only for non-null rows. + Reuse this representation and test across both byte and 64-row boundaries. + +Only `QwpSchemaBinding` needs executable client changes: the bool setter, the +BOOLEAN target-to-wire case and the string parser case. Reuse the existing +buffer ownership, row cancellation, stale-binding guard and duplicate-first +lookup. No production changes to Sender, buffer, encoder, shared serializers, +server ingestion, lookup/cache coordination or protocol. No new conversion +framework, dictionary, fallback or feature switch. + +SOL responsibilities: client implementation plus shared corpora and observable +wire tests; separate real-server E2E implementation/builds; independent and +adversarial source/corpus/test review. Root owns scope, assumption validation, +design/journal and acceptance. Planned corpora exhaust the ASCII case variants +of both accepted words and cover invalid tokens, all nine supplied-boolean +target pairs and explicit missing-value scenarios. Tests must include local +invalid-value errors, duplicate suppression before parsing, mutable input, +reset/clear, A-invalid-B-C row rollback and failed-row-only columns, legacy +accept/reject parity, intentional omission differences and recovered target +values. Use real wire and SQL observations, no reflection or TestOnly hooks. + +The performance skill is applied as in D021/D023: this is functional coverage, +not production optimization, and no hardware bottleneck or speed/allocation-rate +claim is made. Source-level reuse is not a measured performance result. Keep +Maven serialized within each repository, install/freeze the client only after +client gates, then run server gates against that installed JAR without +`-Plocal-client`. Use absolute project-local temp paths; `/tmp` has about 621 MB +free. Preserve dirty worktrees, existing corpora and unrelated server files. +No commits or pushes. + +Initial corpus review caught two ambiguous expectations before acceptance: +numeric target results must say zero/one, not reuse boolean `false`/`true` +labels; and explicit STRING null to BOOLEAN must identify a wire null, not a +stored false bit, even though SQL reads false. Keep the wire and SQL oracles +separate. Root also required the Unicode long-s spelling of `false` as a +rejection case: it specifically detects accidental Unicode case folding, unlike +an arbitrary non-ASCII typo. Packed tests must cross 64 non-null values as well +as 64 total rows; these are different indices when nulls are interspersed. + +The final shared corpora contain 18 primitive-boolean target cases and 80 text +cases: all 50 accepted non-null ASCII spellings, one explicit null and 29 invalid +tokens. Root's independently constructed accepted-token set and target-value +oracle found zero mismatches; independent SOL review agrees. Client wire tests +use 137 rows with 69 supplied booleans, then reset to 70 all-null rows, checking +the raw null bitmap and compacted value bitmap separately. The server pattern +uses 137 rows with 93 supplied values; acceptance requires checking stored values +by row, not only aggregate counts. Float/double zero checks must compare exact +bits so negative zero cannot pass a delta-based assertion for positive zero. + +Final source review keeps the omission comparison deliberately representative: +INT and STRING explicitly contrast schema target nulls with legacy zero/text +`false`. All nine targets still have supplied-value parity tests; the packed +BOOLEAN case and existing numeric-null tests cover their separate missing-value +rules. Do not describe the representative comparison as nine-target omission +coverage. A transient expanded draft was removed before the accepted runs. + +The first client run exposed a raw null-bitmap fixture expectation error; the +first server build exposed missing required WebSocket close callbacks in the +test helpers. Fixes stay in tests, with unexpected close now failing explicitly. +The accepted server invalid-token tests send actual legacy VARCHAR frames and +require both SCHEMA_MISMATCH and the boolean-parse diagnostic. Generic failure +or timeout is not evidence of parser parity. Superseded development logs remain +identified in the run manifests. + +### Final validation and reflection + +All accepted runs passed with zero failures, errors or skips: + +- Client focused: 41 tests, including all three binding test classes. +- Client canonical: 374 selected tests plus 2 packaging checks. +- Server focused: 6 new BOOLEAN E2E tests. +- Server canonical: 299 selected tests. +- Server seeded repeats: 38 tests each with seeds 1/2, 42/43 and + 1234567/7654321. + +The canonical selectors extend D023 with `QwpSchemaBindingBooleanTest` on the +client and `QwpSchemaBooleanE2ETest` on the server; the latter also joins the +three seeded repeats. Commands, selectors and logs are recorded in client +`core/target/schema-boolean-2.11/run-manifest.md` and sibling +`questdb/core/target/schema-boolean.2.11-server/run-manifest.md`. Accepted logs +are client `logs/focused-final.log` and `logs/canonical-final.log`; server +`logs/focused-final.log`, `logs/canonical299.log` and the three +`logs/repeat-*.log` files. The initial server compile-only failure is not counted +as a test run or acceptance evidence. + +Source HEADs remain client `981bdb02a471f3b290c89b8e78cbc422610e329e` and server +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with the dirty schema worktrees +preserved. Runtime is Corretto 25.0.4 with Maven 3.9.11. Server tests use the +standalone client installed after its final gates, never `-Plocal-client`, and +absolute repository-local JVM temp directories. Root verified the final Surefire +classpath, temp path and seed properties, not only the reported counts. + +- Workspace/installed main JAR SHA-256, unchanged during server testing: + `b1aa01621ae22cfc39b15a9b1385c82834f5999ab667fb2c2f6798bdeae4a86f`. +- Both copies of `boolean-to-target.tsv` SHA-256: + `37038ba5b25a254b7b0e005ba449d3d652a5541734a98fdb9f5dc0dae0540581`. +- Both copies of `string-to-boolean.tsv` SHA-256: + `1b8952d51862f68e80c23ea06111b3e4cfdee8a87cf37c3374d57d430b288c44`. + +Root checked that the tracked production diffs in both repositories are +unchanged from the start of this iteration, along with the untracked server +schema-control source. The only executable production changes are in the +already-untracked client binding. Sender, buffer, column writer and all existing +corpora retain their pre-iteration hashes. Both repository diff checks and +explicit checks of the new untracked tests/resources are clean. + +Independent SOL review accepted the conversion code, corpora and final test +contracts; the client author also cross-reviewed the server test, and the server +author independently audited the client implementation adversarially with no +blocker. Runtime gates remain necessary: source review alone did not establish +that test fixtures compiled or that their wire expectations were correct. The +accepted tests now assert actual encoded values, typed local failures, specific +server parsing failures and exact stored results, with no private hooks or +reflection. + +Root accepts iteration 2.11. Reusing strict ASCII comparison and existing target +writers kept production changes limited to a setter, parser branch and type +mapping. No new state or conversion framework was needed, and no performance +improvement is claimed. The SF test proves conversion before persistence and +preservation of recovered values, not byte-identical replay by itself. Ordinary +Sender integration and the remaining D022 conversion/inference/row-boundary +requirements are still unfinished. No commits or pushes were made. + +## D025 — Floating-point source semantics need an explicit compatibility decision + +Date: 2026-09-10. Status: findings verified against source and real-server tests; +characterization accepted. The compatibility decision was subsequently approved +by the user in D026. This entry records the investigation, not implementation. + +The next candidate slice is FLOAT/DOUBLE inputs into BYTE, SHORT, INT, LONG, +FLOAT and DOUBLE. Before assigning implementation, root and three SOL agents +traced ordinary Sender, the column encoder, QWP dispatch and server conversion. +The twelve server-supported numeric pairs are small enough to implement directly +in the binding, but supplied-value parity has two important edge cases. + +### Source evidence and recommendation + +- Ordinary `QwpWebSocketSender.floatColumn`/`doubleColumn` use bitmap-capable + columns and append supplied values without NaN normalization. `ColumnBuffer` + and the column writer preserve the raw value bytes. Crucially, + `QwpColumnWriter.writeNullHeader` emits no bitmap when there are no omitted + values; bitmap capability is not the same as a bitmap on the wire. +- `QwpFixedWidthColumnCursor.advanceRow` checks NaN as a null sentinel only + without a bitmap. With a bitmap, a supplied NaN is non-null and reaches + `WalColumnarRowAppender.checkDoubleToInteger`, which rejects it. Without a + bitmap, BYTE/SHORT instead receive zero and INT/LONG receive SQL NULL. Adding + an omitted row therefore changes whether another row is accepted. D013 + approved a stable source-null rule for LONG minimum only, not for NaN. +- `QwpWalAppender.isFixedTypeCoercionAllowed` confirms the six numeric targets; + DATE and TIMESTAMP are not part of this floating-point slice. For integer + targets, the server casts to long and compares the double round trip, then + checks the narrower integer bounds. Exactly positive `2^63` casts to + `Long.MAX_VALUE`, whose double representation rounds back to `2^63`, so the + equality test accepts it. This is silent clamping of an out-of-range value, + not exact conversion. Fractions and infinities fail the round-trip check. +- FLOAT/DOUBLE targets use normal widening/narrowing; finite DOUBLE overflow + to FLOAT infinity and signed zero are separate behaviors. Do not accidentally + ban them with a blanket finite-value or mathematical-range rule. + +Root recommends NaN-as-source-null for supported numeric targets, consistent +with D013, and mathematical range checking for floating-point inputs to integer +targets. Preserve duplicate-first and supported-pair checks before null handling. +This means zero for BYTE/SHORT and NULL for nullable numeric targets; positive +`2^63` to LONG must fail locally instead of being clamped. Both are proposed +schema-mode exceptions to parity, not changes to legacy Sender or server +ingestion. No per-row state, batch scan, send-time conversion or framework is +needed. The exact negative LONG endpoint remains representable and follows the +target's existing SQL-null sentinel semantics; it is not a new source-null rule. + +One independent reviewer initially called the twelve-pair slice coherent under +existing parity, then withdrew that conclusion after root challenged the +bitmap-free NaN case. A second review initially conflated bitmap-capable storage +with a serialized bitmap and corrected that claim against `writeNullHeader`. +These corrections matter: choosing either always-reject or always-null for NaN +changes one working legacy context. Reviews are evidence to check, not votes. +The independent review favors preserving the clamped endpoint for compatibility; +root favors rejecting it because early validation should not silently change +an out-of-range integer. User approval must settle the deliberate exception. + +SOL server validation owns test-only real-server characterization using ordinary +Sender and exact stored values or server error messages. Root verified the +final tests and logs before presenting the decision. No client build/install, +server production change, ordinary Sender integration or new converter is part +of this investigation. The performance skill remains a functional-work guard: +no optimization, hardware bottleneck or speed/allocation-rate claim is made. +Existing worktrees and the frozen 2.11 client artifact are preserved. + +### Characterization construction and review + +Two tests were added to the existing server `QwpSenderE2ETest`, exercising +ordinary public Sender calls. NaN covers both FLOAT/DOUBLE sources and all four +integer targets, with and without an omitted row. The endpoint test checks +exactly positive `2^63`, the adjacent smaller representable source value and +the adjacent larger value, separately for FLOAT and DOUBLE. Assertions use +exact SQL values, missing-value predicates, specific SCHEMA_MISMATCH diagnostics +and zero committed rows from rejected blocks. + +The first endpoint fixture mixed FLOAT and DOUBLE setters in one legacy layout +and failed locally, before the server conversion. Root caught the invalid +assumption; separate table layouts corrected the test. Root and adversarial +review also required explicit batching settings for NaN plus omission, so a +timer cannot flush the NaN row before the omitted row is appended. A dedicated +test helper pins the row threshold above two, disables the byte trigger and +sets the interval to the largest supported value. It does not alter the shared +rejection helper or production code. The first compile after this change used +a private superclass timeout constant; an explicit test timeout corrected it. +Error-message target names use locale-independent case conversion. Failed and +superseded intermediate runs are retained and are not final acceptance evidence. + +### Final verification and handoff + +The final two characterization tests passed, with zero failures, errors or +skips. Three repeats also passed both tests, using seeds 1/2, 42/43 and +1234567/7654321. Accepted server logs are +`core/target/schema-floating-feasibility/logs/final.log` and the three matching +`logs/repeat-*.log` files. The adjacent `run-manifest.md` records the selector, +JVM arguments, temporary paths and superseded runs. An intermediate compile +was interrupted for the locale fix; it is explicitly excluded from acceptance. + +Root checked the final source, all four successful logs, the last Surefire XML +classpath/temp/seed properties and unchanged workspace/installed client JAR. +The final server source received independent adversarial SOL acceptance. No +reflection, TestOnly method or private-state hook is used. These checks establish +the two legacy behaviors; they do not validate an unimplemented schema converter +or replace its future full regression matrix. + +- Client JAR SHA-256 remains + `b1aa01621ae22cfc39b15a9b1385c82834f5999ab667fb2c2f6798bdeae4a86f`. +- Final `QwpSenderE2ETest.java` SHA-256: + `9bcb321488b05e724eb90c8e5eb3f4ea205bec3b5716f7380fbe6fe8668a5ba6`. +- Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD + remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`. Tracked production diff + hashes in both worktrees match the pre-investigation values, and the client + binding is unchanged. Both repository diff checks are clean; earlier dirty + work and the unrelated server Rust checkout are preserved. + +Root accepts the characterization, not iteration 2.12. The design explicitly +keeps the two proposed parity exceptions pending. No conversion code, Sender +activation, legacy behavior, protocol, persisted bytes or server ingestion +was changed. No client reinstall, full canonical rerun, commit or push was +performed. Next action requires the user's decision on consistent NaN missing +values and rejection of floating-point inputs outside an integer target's range. + +## D026 — Approved FLOAT/DOUBLE numeric conversion rules + +Date: 2026-09-10. Status: user approved; iteration 2.12 accepted after review and +client/server validation. + +The user answered "yes" to applying both recommended rules in schema mode while +leaving legacy behavior unchanged. NaN is source missing for supported numeric +targets, and floating-point input outside an integer target's mathematical range +fails locally. Design revision 26 makes both exceptions explicit; D025's +characterization remains the preserved legacy baseline. + +Scope: add `floatColumn` and `doubleColumn` to the non-owning binding for BYTE, +SHORT, INT, LONG, FLOAT and DOUBLE. No text formatting/parsing, decimals, arrays, +DATE/TIMESTAMP conversion, inference, row-boundary adoption or Sender activation. +Executable production changes stay in `QwpSchemaBinding`; existing buffer, +encoder, ownership and rollback remain. One shared floating numeric helper is +acceptable: widening a FLOAT input to DOUBLE is exact for all non-NaN values, +and NaNs are normalized before append. Do not duplicate twelve target branches +or introduce a converter framework merely to keep separate setter names. + +Preserve duplicate-first behavior, exact supported-pair/parameter/designated +checks before NaN normalization, and the input type in error context. NaN uses +the target missing representation, not a raw NaN payload. Integer conversion +rejects fractions and infinities and explicitly excludes positive `2^63` before +any long cast could silently saturate. Inclusive BYTE/SHORT/INT ranges and the +negative LONG endpoint remain. A finite conversion landing on INT/LONG minimum +is still a present target sentinel that reads SQL NULL, not a new source-null +policy. Floating targets retain server-equivalent non-NaN bits and rounding, +including negative zero and overflow/underflow when narrowing DOUBLE to FLOAT. + +SOL responsibilities: client binding, shared raw-bit corpus and observable +client tests; separate real-server E2E author/build owner; independent source, +corpus and adversarial test reviewer. Root owns scope, independent boundary +oracle, design/journal and acceptance. The client corpus owner publishes its +format early so server tests can be written independently. Tests must assert +wire type, actual bitmap/value bytes and stored values separately; a cursor's +sentinel interpretation alone does not prove an explicit wire null. + +Cover all twelve pairs; +/-zero, subnormal and finite extrema; integer edges +and adjacent representable floats; fractions and infinities; multiple NaN +payloads/signs; nulls with and without other omitted rows; duplicate invalid or +unsupported inputs; reset/clear/stale bindings; whole-row rollback preserving +A/C and removing failed-row-only columns; and converted bytes persisted and +recovered through SF. Legacy tests must separate preserved parity from the two +approved differences. No reflection, private hooks, TestOnly APIs or assertions +whose expected result is computed by the converter being tested. + +Run focused and canonical client gates before installing/freezing its standalone +artifact. Only then run server gates against that installed JAR, never +`-Plocal-client`. Extend the 2.11 canonical selectors with the new floating +tests and retain both D025 legacy characterization methods. Repeat the server +schema matrix with the existing three seed pairs. Keep Maven serialized within +each repository and use absolute project-local JVM temp directories. Preserve +dirty worktrees, all prior corpora and unrelated files; no commits or pushes. + +As in D024, the performance skill is used for functional-work discipline rather +than a production optimization: existing storage/writers are reused, measured +hardware behavior is unknown, and no bottleneck, throughput or allocation-rate +claim is made. Functional test counts are not performance evidence. + +### Implementation and independent checks + +The implementation uses two public setter methods and one shared floating +numeric path. It reuses the numeric target predicate with LONG conversion; +following root review, SOL consolidated the two identical allowlists. No buffer, +encoder, wire mapping, owner or server ingestion change was needed. Integer +checks use finite/whole-value validation and exact mathematical bounds, with +the LONG upper bound expressed as exclusive positive `2^63`. + +The shared corpus is frozen at 350 raw-bit cases, SHA-256 +`7194ec96006341cf5a5d9081fd75d8f1ab7a1a5813067ba80971d01df8957216`. +Its seven fields separate input bits, target type, wire type, wire bits and +stored value. Explicit wire null is distinct from a present INT/LONG sentinel +and from BYTE/SHORT's stored zero. Stored floating values use raw-bit expectations +to preserve negative zero and avoid relying on SQL text formatting. + +Root's exact-IEEE BigInt oracle found eight incorrect draft SQL expectations: +NaN-to-BYTE/SHORT encodes an explicit null but stores zero, not SQL NULL. These +were corrected before acceptance. Additional vectors cover signaling NaN, +negative fractions, LONG/INT boundaries and adjacent representable inputs, +round-to-even ties, zero/subnormal and subnormal/normal transitions, and the +finite-to-infinity narrowing threshold. Input raw bits, not approximate numeric +case labels, are authoritative: FLOAT cannot distinguish integer values already +rounded to the same input before the setter call. + +Both root's independent BigInt oracle and a separate SOL arithmetic check agree +on all 350 cases: 106 invalid, 36 explicit missing and 208 present values across +all twelve input/target pairs. Root's diagnostic script is retained at +`core/target/schema-floating-oracle-2.12/check-corpus.cjs`; its rounding logic +was separately cross-checked on 10,000 deterministic IEEE samples. That sample +validates the diagnostic oracle, not 10,000 executions of the new converter. + +Root and SOL review strengthened duplicate tests to assert retained encoded +values rather than only successful encoding. The new client tests also check +actual null bitmap/value bytes, whole-row and failed-column rollback, reset and +stale binding behavior, unsupported targets before NaN handling, and input/target +error context. Independent client review found no remaining issue. The first +focused run passed 47 tests, including all six new floating tests; broader +client and real-server validation remain the acceptance gates below. + +Server review required three additional observations before runtime acceptance: +test all 104 legacy-invalid raw frames, excluding only the two approved positive +`2^63` clamps; cover both NaN row contexts for both input types and all six +targets; and prove that a bitmap actually exists and contains zero compact +values for an all-missing column. A cursor null flag alone was insufficient, +because its bitmap-free sentinel interpretation is the original compatibility +problem. Once bitmap presence and compact value counts are checked, the existing +cursor's row flags and value reads can be reused without another wire parser. + +Stored-value checks were corrected to recognize NaN as FLOAT/DOUBLE SQL null; +BYTE/SHORT omission cases assert actual zeros, not just non-null counts. The SF +test uses literal expected integer and floating bits before persistence and +after recovery, rather than describing converted values as byte-identity proof. +Independent server source review accepted these final contracts, with runtime +verification still required. + +The first server focused run exposed an incorrect sparse-layout column index +in the new NaN fixture: only `value` was written, so its wire index is zero even +though it follows `case_id` in the SQL schema. Root caught the assumption during +review; the fixture now asserts the single encoded column's name and type. The +next focused run exposed an expected SQL-header mismatch for two unaliased +counts. These are test-fixture failures, not converter defects. Their logs are +retained separately from the final accepted runs. Source review did not replace +compilation and execution. + +### Final validation and acceptance + +All final gates passed with zero failures, errors or skips: + +- Client focused: 47 tests, including 6 new floating tests. +- Client canonical: 380 tests plus 2 packaging checks. +- Server focused: 5 new floating E2E tests. +- Server canonical: 306 tests. +- Server seeded repeats: 45 tests each at seeds 1/2, 42/43 and + 1234567/7654321. + +The client selector extends D024's 374 with `QwpSchemaBindingFloatingTest`. +The server selector extends D024's 299 with the five-test +`QwpSchemaFloatingNumericE2ETest` and both D025 legacy characterization methods +in `QwpSenderE2ETest`. The repeat selector likewise adds those seven tests to +D024's 38. Explicit method filters retain both older UUID checks in the +canonical run; adding methods to the source alone would not select them. + +Client commands and logs are recorded in +`core/target/schema-floating-2.12/run-manifest.md`; accepted logs are +`logs/focused-initial.log`, `logs/canonical-final.log` and `logs/install.log`. +Server commands and logs are recorded under sibling +`questdb/core/target/schema-floating.2.12-server`; accepted logs are +`logs/focused-final2.log`, `logs/canonical306.log` and the three +`logs/repeat-*.log` files. The two failed focused fixture runs remain separate. + +The shared 350-case corpus is byte-identical in both repositories. Server tests +consume every wire and stored-value field: 244 accepted cases, plus all 106 +local invalid cases; 104 of those also receive specific legacy-frame NACKs, +while the two deliberately different LONG clamps are covered by D025. The NaN +matrix checks both inputs, all six targets and both row contexts. SF validates +target-native values before persistence and after recovery; byte-identity replay +remains covered by the existing replay suites, not inferred from this test. + +Root verified the successful logs, printed seed values, Surefire's installed-JAR +classpath and project-local temp paths, and final artifact identity. Server +tests use the frozen standalone client without `-Plocal-client`; no reinstall +occurred during server testing. Runtime is Corretto 25.0.4 with Maven 3.9.11. + +- Workspace/installed client JAR SHA-256: + `8e6dfd80195889b30eb1d0a8466f5529a15a242bc19b1d6814b4ff4ee988497e`. +- Client `QwpSchemaBinding.java` SHA-256: + `7ca784dc29c76d1ace2e2ebafef963c5e2d77939c485fdd5c2d04c104b4be53f`. +- Server `QwpSchemaFloatingNumericE2ETest.java` SHA-256: + `178815e8e7dc046f0bcd1e76926047c0bf628d1b791345165e460833dc8389ee`. +- Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD + remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`. + +Tracked production diff hashes in both repositories, the shared buffer/writer/ +Sender hashes, the server schema-control source and all older corpora remain +unchanged. Only the previously untracked client binding has new executable +production changes in this iteration. Both tracked diff checks and explicit +new-file whitespace checks are clean. Dirty worktrees and unrelated server +Rust files are preserved; no commits or pushes were made. + +Root accepts iteration 2.12. Independent SOL review accepted the client code, +corpus and tests, and the final server test contract; the server author also +audited the client conversion independently. Review strengthened evidence without +adding production state or a conversion framework. The performance skill kept +functional coverage separate from unmeasured performance claims. Ordinary +Sender activation, remaining conversion families, inference and producer-owned +row-boundary adoption are still deferred under D022. This is a completed +conversion component, not an activated or release-complete schema-aware Sender. + +## D027 — STRING-to-numeric conversion, characterized before implementation + +Date: 2026-09-10; accepted 2026-09-11. Status: iteration 2.13 accepted after +explicit LONG text overflow approval, review and client/server validation. + +The next bounded slice adds STRING input to BYTE, SHORT, INT, LONG, FLOAT and +DOUBLE in the existing non-owning binding. It directly supports early rejection +of malformed or out-of-range values. Ordinary Sender activation, inference, +row-boundary adoption, numeric formatting and other conversion families remain +outside this increment. D022's integration gate is unchanged. + +Start with real-server characterization using the previously accepted client +artifact. Similar parser implementations do not prove parity: the client reads +Java characters while ingestion reads encoded UTF-8. Explicitly test grammar, +Unicode, malformed surrogates and NUL as well as numeric boundaries. Do not +assume every non-ASCII input is rejected. Server acceptance and stored values +must be established before releasing the production patch. + +Reuse direct target parsers and the existing column append/rollback machinery. +Do not route integer text through DOUBLE, add a converter framework, introduce +another buffer owner or change the legacy Sender/server to fit client behavior. +Preserve duplicate-first checks and existing target/parameter/designated guards. +Any newly discovered contract difference requiring a product choice must be +reported before silently changing acceptance. + +SOL ownership is split between client binding/shared corpus/client tests, +server characterization/E2E/builds, and independent adversarial review. Root +owns architecture, independent evidence checks and this journal. Acceptance +requires shared cases with exact target wire values and SQL results, server +rejection parity, null/sentinel contexts, duplicate handling, partial-row +rollback preserving completed rows, and converted SF recovery. No reflection, +private hooks or test-only production APIs. Maven runs remain serialized per +repository and the installed standalone client is frozen during server gates. + +The performance skill was read with both evidence references. This is a +functional extension, not a production optimization: no bottleneck is claimed +and no measured throughput/latency comparison is available. Its optimization +gate does not establish correctness for this work. Keep implementation scope +small and make no performance claim. Use absolute repository-local test temp +paths because shared `/tmp` is close to capacity; preserve unrelated files. + +### Characterization findings before the production gate + +Source review disproved the initial SHORT simplification: server `parseShort` +accepts only optional minus and ASCII digits, while `parseInt` accepts leading +plus and underscores. A strict grammar check followed by the existing integer +parser and short range check can preserve acceptance without another arithmetic +parser. This is a simplicity choice, not a measured performance improvement. + +Root also reproduced an existing LONG parser overflow through the public +`Numbers.parseLong` API in the accepted 2.12 client artifact: +`"21000000000000000000"` becomes `2553255926290448384`; its negative counterpart +becomes the negative of that value. Values such as unsigned 64-bit maximum +reject, so checking only obvious boundary strings would miss the problem. +Client and server source use the same insufficient wrap-detection arithmetic. +Real-server ingestion confirmation is being run before a product recommendation. + +Independent SOL review agrees that silently adding a checked parser would +violate the current supplied-value parity rule. Root recommends rejecting +mathematically overflowing LONG text in schema mode if ingestion confirms the +behavior, while preserving legacy ingestion and shared parsing utilities. +That would require an explicit compatibility decision; production edits remain +withheld. Preserve other established grammar independently of this arithmetic +decision, including LONG's existing suffix/separator behavior. + +The real-server test confirmed all five supplied wrapping examples (positive +and negative 21e18, positive 25e18, 42e18 and 63e18). It sends legacy VARCHAR +wire values, requires successful write responses and checks the exact stored +LONG values. `testLegacyLongParserWrapCharacterization` passed 1/1 with zero +failures/errors/skips; evidence is in sibling +`questdb/core/target/schema-string-numeric.2.13-server/logs/wrap-characterization.log`. + +The broader diagnostic characterization passed separately in +`logs/characterization2.log`. This records acceptance and stored values rather +than asserting a frozen conformance corpus; it is not converter acceptance. +Its first run, retained as `logs/characterization.log`, stopped on an incorrect +test assumption that every rejection used one status code. Production behavior +was not changed to make the diagnostic pass. + +Root verified both successful logs and the unchanged standalone 2.12 artifact +SHA-256 `8e6dfd80195889b30eb1d0a8466f5529a15a242bc19b1d6814b4ff4ee988497e`. +Client binding, client tracked production diff, server tracked production diff +and server schema-control hashes also remain unchanged. No converter patch or +ordinary Sender activation has landed in 2.13. The user has been asked whether +to approve strict LONG text range rejection in schema mode. This iteration +remains incomplete pending that decision; the dedicated test preserves the +legacy baseline independently of the future choice. + +The server build owner recorded commands and scope in +`core/target/schema-string-numeric.2.13-server/characterization-manifest.md`. +The diagnostic matrix has 366 observations, not a full grammar proof. Root +corrected a summary overstatement: rejection of `1L` and `0x10` does not prove +rejection of floating suffixes or complete hexadecimal floating literals. +Those remain explicit cases for the future shared corpus. All agents have +paused; no client source, tests, resources or build artifacts changed in 2.13. + +### Approved LONG text range exception and resumed implementation + +The user answered "Yes, proceed" to rejecting mathematically overflowing LONG +text in schema mode while leaving legacy behavior unchanged. Revision 27 records +this exception to supplied-value parity. Implementation is now released, not +accepted: all 2.13 converter and regression gates remain to be run. + +Use a private checked LONG text parser in the existing binding if necessary; +do not change shared `Numbers`, ordinary Sender, or server ingestion. Preserve +the existing syntax independently of range checks: no leading plus, existing +underscore/DEL separator rules and terminal `L`/`l`. Arbitrary leading zeros +must not cause false overflow. Exact signed-long endpoints remain accepted; +the minimum is a supplied target sentinel, not a missing STRING input. Do not +route parsed values through the LONG or floating setters, whose source-null +rules apply to different input types. + +Reactivated SOL ownership remains client implementation/corpus/tests, server +E2E/builds, and independent adversarial review. Root will independently check +the checked arithmetic and acceptance evidence. The skill's optimization gate +does not apply cleanly to this requested functional extension; its evidence +discipline is retained without optimization edits or performance claims. Shared +`/tmp` now has roughly 211 MB free; all new test JVMs must use repository-local +temporary directories. No unrelated cleanup, commits, or pushes are authorized. + +### Implementation review and independent checks + +The first candidate adds direct numeric writes and small private SHORT/LONG +parsers to the existing binding. Numeric parse errors share one conversion +boundary; no additional owner, cache, protocol or delivery state is introduced. +SHORT checks its strict syntax before the existing INT parser/range check. +LONG uses checked negative accumulation so both signed-long endpoints remain +representable without overflowing intermediate arithmetic. + +Root compiled the candidate separately from Maven outputs and exercised the +public binding and serialized LONG wire against an independent regex/BigInteger +oracle. All 14,598 deterministic cases passed (2,921 accepted, 11,677 rejected; +seed 2132026), including boundary neighborhoods, known wrapped values, syntax +mutations, Unicode and 10,000 leading zeros. Candidate binding SHA-256: +`724834280cffa6b43d8b229513211c97c9df89d7f55390e4e841dc19511477d1`. +The same diagnostic had failed against the old 2.12 artifact with the expected +missing-conversion error. Final acceptance must rerun against the frozen new +artifact; this isolated compile does not replace Maven or server validation. + +Independent SOL review and root both caught two draft test weaknesses before +acceptance: BOOLEAN text `"2"` is INVALID_VALUE, not an unsupported pair, and +the mutable-input test discarded its first row before observing its converted +value. The author was asked to correct those assertions and format new test +helpers as readable code. Tests must prove their named behavior. + +The legacy NBSP rejection exposed a pre-existing server diagnostic bug, not a +new conversion contract: after numeric parsing fails for `"\u00a01"`, error +logging throws `LogError: Invalid UTF-8` and the response is INTERNAL_ERROR +instead of SCHEMA_MISMATCH. The earlier characterization log records this at +the numeric-error logging path. Keep this server behavior outside the patch; +tests must identify this specific observed response rather than accepting an +arbitrary failure status. Local conversion still rejects the malformed value. + +Runtime checks exposed another load-bearing difference: the server's UTF-8 +memory parsers accept both `f` and `d` suffixes for decimal FLOAT/DOUBLE text, +but the client character parsers accept only their own decimal suffix. The +initial corpus correctly expected server acceptance for FLOAT `"1d"` and DOUBLE +`"1f"`; the binding failed those cases. This was a production compatibility +defect, not a reason to weaken the corpus. Source equality with the server's +character parser had not established parity with its ingestion memory parser. + +The fix remains binding-local: find an opposite numeric suffix before trailing +ASCII whitespace, require a preceding digit or decimal point, then pass a slice +of the original CharSequence to the target parser. It does not copy/rewrite +text, use an exception for a successful conversion, change shared parsers or +add state. The prefix parser validates the remaining grammar. Explicit cases +cover complete hex forms, suffix case, trailing dots, NUL/whitespace, malformed +double suffixes and suffixed NaN/Infinity. No new compatibility exception is +needed; this restores the agreed server behavior. + +Root exercised this candidate binding and serialized floating values against +the actual server UTF-8 memory parsers over 16,628 deterministic inputs (2,806 +accepted, 13,822 rejected; seed 2132027), with zero mismatches. This is a +public-API/parser differential diagnostic, not a real-server ingestion test or +an independent implementation of floating arithmetic. The separate JDK and +BigInteger corpus audit checks 429 cases: all 255 non-null integer acceptance/ +value expectations, 111 accepted floating bit patterns and 6 explicit nulls; +the 57 rejected floating cases only receive structural marker checks there. +Real-server tests own the floating rejection-policy evidence. + +The final candidate corpus has 429 cases (188 accepted, 241 locally invalid), +including the five explicit legacy wrapping exceptions. Client review is +accepted at binding SHA-256 +`a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215` +and corpus SHA-256 +`e1b3728510afd915bd1cf589ff2c2900337614e331ff821d1014cbb6c825c9d1`. +Client focused aggregate passed 53 tests plus 2 packaging checks; canonical +passed 386 tests plus 2 packaging checks. The initial failing fixture/parser +runs remain distinct from the accepted logs. Server ingestion, recovery and +final artifact identity gates are still pending at this point. + +Final source review strengthened the server tests without changing production: +replace the print-based characterization with fixed corpus assertions; require +exactly 236 legacy rejections after the five separately asserted wrap cases; +distinguish the known NBSP logging error from numeric parse/range failures; +check literal sentinel/default values and cross-suffix values through SF. +The encoded identity is compared to the actual DESCRIBE snapshot, not a +synthetic fixture constant. The final focused rerun after that identity check +passed all six tests; the broader server gate passed 312 tests with zero +failures/errors/skips. Three seeded repeats remain before final acceptance. + +Root reran both generated diagnostics against the frozen installed artifact, +not the isolated candidate classes: all 31,226 public-binding/wire cases passed. +The standalone artifact SHA-256 is +`5c1d2a24e6147a066a9fbc2e28e4fdf8faaca1b6baece62af7d1769c02b71144`. +Exact diagnostic commands, scopes and limitations are recorded in +`core/target/schema-string-numeric-2.13/oracle-manifest.md`. + +Reflection: reuse of an existing parser is the right starting point, but +character versus byte entry points can implement different grammar. Test the +actual ingestion path before claiming parity. The two necessary adaptations +stay local and explicit (checked LONG arithmetic and floating suffix handling), +with a strict SHORT grammar check. No conversion registry, compatibility mode, +alternate buffer owner, shared-parser refactor or send-time transformation was +needed. This slice reduces conversion gaps; it does not resolve D022's Sender +row-lifecycle, inference and activation obligations. + +### Final validation and acceptance — 2026-09-11 + +All final gates passed with zero failures, errors or skips: + +- Client focused aggregate: 53 tests plus 2 packaging checks. +- Client canonical: 386 tests plus 2 packaging checks. +- Server focused: 6 tests, rerun after the final schema-identity assertion. +- Server canonical: 312 tests. +- Server repeats: 51 tests each at seeds 1/2, 42/43 and 1234567/7654321. + +The selectors extend the accepted 2.12 gates with +`QwpSchemaBindingStringNumericTest` on the client and +`QwpSchemaStringNumericE2ETest` on the server. Existing conversion, negotiation, +feedback, replay and legacy characterization coverage remains selected. +The shared 429-case corpus is byte-identical in both repositories: all 188 +accepted cases have exact target wire and stored-value checks, all 241 invalid +cases fail locally, 236 also receive the specified legacy rejection, and the +five approved overflow differences have separate exact legacy stored-value +assertions. The Unicode logging failure is specifically asserted, not treated +as permission to accept arbitrary server errors. + +The null matrix distinguishes present parsed INT/LONG sentinels and floating +NaN from explicit bitmap nulls, with and without another omitted row. BYTE and +SHORT missing values are asserted as exact SQL zero. The SF case checks +conversion before persistence and recovered INT 42, FLOAT `"0.1d"` bits +`0x3dcccccd`, and DOUBLE `"-0.0f"` bits `0x8000000000000000`. It does not replace +the existing generic replay byte-identity evidence. + +Client commands/logs: `core/target/schema-string-numeric-2.13/run-manifest.md`, +with final `logs/focused-final.log`, `logs/canonical-final.log` and +`logs/install-final.log`. Server commands/logs: sibling +`questdb/core/target/schema-string-numeric.2.13-server/run-manifest.md`, with +final `logs/focused-final2.log`, `logs/canonical312.log` and all three +`logs/repeat-*.log`. Diagnostic characterization and superseded fixture/parser +runs remain separately recorded. They are not final acceptance evidence. + +Root verified the successful log summaries, final Surefire seeds and absolute +project-local temp path, the installed standalone client on the server test +classpath, and unchanged artifact identity. There was no `-Plocal-client` or +client reinstall during server gates. Runtime: Corretto 25.0.4, Maven 3.9.11. + +- Workspace/installed client JAR SHA-256: + `5c1d2a24e6147a066a9fbc2e28e4fdf8faaca1b6baece62af7d1769c02b71144`. +- Client binding SHA-256: + `a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215`. +- Shared corpus SHA-256: + `e1b3728510afd915bd1cf589ff2c2900337614e331ff821d1014cbb6c825c9d1`. +- Client test SHA-256: + `a475aafdd1c9863d1f04722751a5af3d1296ecac81afdc20d2b9a1c1cb09eb55`. +- Server test SHA-256: + `b5f5f04f118d032cc2da5cea6c7cdbf3de911ec87aa65e8b66703241cb3c3444`. +- Client HEAD: `981bdb02a471f3b290c89b8e78cbc422610e329e`. +- Server HEAD: `12a33d651e51e2682e7a448c8db5168fc72dfad3`. + +Both SOL source reviews accepted the final server test identity; the independent +reviewer also accepted the final client binding, corpus and tests. Root's +generated diagnostics passed again against the final installed artifact. +Tracked production diff hashes in both repositories, shared buffer/writer/ +Sender hashes and server schema-control hash are unchanged from 2.12. The only +new executable production behavior is in the existing untracked binding. +Tracked diff and explicit new-file whitespace checks are clean. No unrelated +files were removed, no shared parser or server production code was changed, +and no commits or pushes were made. + +Root accepts iteration 2.13 as a production-quality conversion component. +The performance skill kept functional verification separate from unmeasured +performance claims; no optimization or hardware-bottleneck claim is made. +Ordinary Sender conversion, remaining conversion families, inference and +automatic row-boundary adoption remain deferred under D022. This is not an +activated or release-complete schema-aware Sender. + +## D028 — Complete missing-conversion inventory — 2026-09-11 + +### Request and scope + +User requested all missing conversions in the design, cross-checked against +server source. This is a documentation-only audit after accepted iteration +2.13, not another implementation iteration. Root owns the document changes; +three SOL agents independently inspected scalar dispatch, complex types and +the public client/binding surface. No production code or tests were changed, +and no builds, commits or pushes were requested. + +### Decisions and findings + +Design revision 28 adds a complete public-input-to-target inventory, including +missing identity setters, parameterized target representations and wire-only +DATE coverage. "Implemented" means the existing binding component, not ordinary +Sender activation. Null acceptance, ignored duplicates and no-ops cannot be +used to claim support for effective non-null values. + +The inventory follows the actual wire cursor, dispatcher and value routine, +not general SQL casting or helper names. STRING input emits VARCHAR; no separate +TYPE_STRING exists. BYTE/SHORT/INT are genuine public input types, not LONG +aliases. SYMBOL is dictionary-backed. Typed decimal and geohash text overloads +parse locally before producing their respective wire types and are not the +same conversion path as stringColumn. + +Missing work includes numeric/date/timestamp/text/decimal cross-pairs, remaining +string parsers, native setters for CHAR/IPv4/LONG256/BINARY, all decimal/geohash/ +array bindings, and numeric/special-type text formatting. Source-specific null +rules remain explicit: approved numeric source-null exceptions do not silently +decide new families, and decimal cursors use bitmap-only null detection rather +than the fixed-width cursor's sentinel inference. + +The source audit found five boundaries that must stay visible until explicitly +resolved: BINARY reaching text parsers; existing-array element/rank validation +gaps despite LONG_ARRAY auto-create rejection; non-ASCII CHAR text corruption +and UTF-16 surrogate representation; designated timestamp fixed-width fallback; +and decimal auto-create scale checks that rely on assertions. These are recorded +as source-proven paths requiring observable tests and a compatibility decision, +not as approved casts, approved behaviour changes or runtime-confirmed failures. + +Corrected the earlier array coverage wording: the server copies elements; it +does not establish LONG/DOUBLE element conversion. Keep missing-table/column +inference, designated row completion and automatic row-boundary adoption apart +from the conversion matrix. Do not add a conversion framework, fallback, +send-time transformation or public DATE setter simply to fill out the table. + +### Source verification and review + +The design contains a linked source map with methods and snapshot line numbers. +Both repositories contain existing uncommitted feature work, so base commits +alone are not presented as the audited implementation: + +- Client base: `981bdb02a471f3b290c89b8e78cbc422610e329e`. +- Server base: `12a33d651e51e2682e7a448c8db5168fc72dfad3`. +- Client binding SHA-256: + `a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215`. +- Server QwpWalAppender SHA-256: + `9d61e060177bae1c10f507f60dc98443fe9ce350d9c0a18790935f58695e5d40`. +- Server WalColumnarRowAppender SHA-256: + `61d80aa814f0674be3aa655f65e8b9570da98b261d6d25c7123bccfcc1cc71b0`. + +Root cross-checked the entire target dispatcher, cursor factory, binding +entrypoints/target selection and the cited parser/formatter boundaries. Root +corrected initial audit assumptions about a nonexistent TYPE_STRING and the +actual UTF-8 sink implementation. Independent draft review corrected the binary +overload list and added decimal-specific sentinel coverage. The public-client +review accepted the corrected inventory without remaining findings; the complex +type review accepted the final corrections and found no missing family or +overbroad target pair. All linked source paths resolve. Tracked diff and explicit +new-document whitespace checks are clean. Client/server tracked production diff +hashes, the binding, ordinary Sender and both audited server appenders are +unchanged from the start of the audit. + +No new runtime coverage is claimed. Iteration 2.13's recorded acceptance remains +the latest test evidence; this audit does not make missing conversions or the +ordinary Sender release-ready. + +## D029 — BINARY target conversion (iteration 2.14) — 2026-09-11 + +Status: accepted as a conversion component; ordinary Sender activation remains deferred. + +### Scope and rationale + +After the conversion audit, the user asked to proceed incrementally. Root chose +BINARY target support: the three existing binary input forms (byte array, +DirectByteSlice, native pointer/length) and stringColumn into BINARY. This is a +bounded family with an existing buffer representation and no new parser. The +unresolved BINARY-to-number/date/etc. parser paths are explicitly excluded, +along with ordinary Sender activation and all other conversion families. + +The existing ColumnBuffer supports BINARY and VARCHAR using the same offsets +and byte storage. Use addBinary for opaque input and addString for existing +UTF-8 encoding; select TYPE_BINARY before append. Add no second buffer owner, +temporary converted byte array, shared parser change or send-time conversion. +The binding remains non-owning and the caller retains row completion/rollback. + +Effective null byte-array/slice arguments are INVALID_VALUE, not implicit nulls. +Native lengths must be 0..Integer.MAX_VALUE, with a nonzero pointer required +only for nonempty input. Reject out-of-range lengths before payload growth or +source-memory access. The documented caller obligation to provide readable native memory +remains; the client cannot validate arbitrary addresses. A zero pointer with +zero length is an empty value. Apply duplicate suppression before argument +validation, as already required for schema-mode setters. + +String null and omission use the target null representation. Empty bytes stay +distinct from null. Root verified that current server MemoryCARW.putBin and +MemoryPARWImpl.putBin preserve zero length, despite stale public Sender Javadoc +saying empty becomes null. That existing documentation discrepancy is recorded, +not used as a reason to copy incorrect behaviour or change ordinary Sender. + +### Validation and ownership plan + +SOL client owner implements the binding, shared literal-byte corpus and public +component tests. SOL server owner independently checks the server contract and +adds real-server discovery/conversion/wire/SQL/recovery tests. A third SOL agent +reviews assumptions and then the implementation adversarially. Root owns the +design/journal, scope decisions, source cross-checks and final acceptance. + +Tests must cover every input form, all byte values, UTF-8 edge cases and lone +surrogates, null versus empty, bitmap boundaries, immediate copying of mutable +or subsequently freed input, ignored invalid duplicates, typed local errors, +rollback retaining completed rows and removing failed-row-only definitions, +reset/clear, exact pinned schema identity and TYPE_BINARY, compression and SF +recovery. Shared vectors carry literal expected bytes; legacy comparisons must +check actual stored bytes. No reflection, private-state inspection or new +test-only production hooks. Oversized native-length tests must reject before +accessing deliberately unusable memory, never allocate multi-gigabyte payloads. + +Maven execution and client installation are serialized. Client focused tests +precede independent review and the extended canonical gate; only the approved +standalone artifact is installed for server focused/canonical/seeded gates. +No server production, shared buffer/writer or ordinary Sender changes are +planned. Previous dirty work and unrelated server parquet files are preserved. + +The performance skill was found under its installed package path after the +catalog path failed. Its measured bottleneck/optimization workflow does not +provide a baseline for this newly supported functional path. Root uses source +copy/allocation inspection and correctness validation here, makes no measured +performance claim, and does not optimize existing shared hot paths. Source +inspection confirms that the existing buffer directly encodes UTF-8 and copies +native input during the call; tests must verify the ownership consequences. + +### Starting source identity + +Base commits remain those recorded in D028. The starting binding SHA-256 is +`a54f9ad6a68ed1d180c5d7a63599be6dcb69750ca7903ee9ef5411ed58501215`. +Protected tracked production diffs remain: + +- Client: `ab7c4ef0ee366553ec27bbdbe7cf3a517eff44c2fdba15070320fb8a786378af`. +- Server: `cef922f159d0b62fc72cdf330b3fa70f41024568e9de26c17cbaf808c4e422ce`. + +QwpTableBuffer, QwpColumnWriter, ordinary Sender and the two server appenders +match their D027/D028 hashes. The latest accepted tests remain iteration 2.13 +until the final results below are recorded. + +### Adversarial findings and narrow scope adjustment + +The independent corpus review caught a copied-data error: the draft +binary_all_256 vector had only 242 bytes, and both expected fields repeated the +mistake. The corrected vector is exactly 00..ff, with an independent length and +byte-by-byte progression assertion. Draft exact-target enforcement was also +corrected to reuse the binding's existing expected-target helper; BINARY input +must not silently enter VARCHAR merely because the storage layout is shared. + +The requested partial-encoding test then found a real shared-buffer bug. +`focused-second.log` ran 7 tests with one failure: expected byte 1, observed +120 (the injected UTF-8 prefix `x`). On a pre-existing column, addString had +already appended bytes when the CharSequence threw, but had not incremented +size or published the terminal offset. cancelCurrentRow skipped the column +because its size still equaled the completed row count. An explicit truncateTo +at the same size would also be a no-op. Failed-row-only columns were removed +correctly; that case alone would have hidden the existing-column failure. + +Root and both SOL reviewers confirmed that a binding-only repair would need a +new internal checkpoint API or temporary conversion buffer. Root instead +authorizes one narrow shared-buffer correction required to complete this slice: +make addString and both addBinary variable-width appends exception-atomic by +restoring their data and offset positions on RuntimeException/Error. Publish +value and row counts only after success. The guarded region includes encoding/ +copying, checked-offset validation and offset publication/growth; observable +tests inject STRING prefix failures, including after data-buffer growth. Binary +copy/allocation and offset-buffer failures are source-reviewed, not fault-injected. +No new per-column state, callbacks, rollback scan, buffer owner or shared-memory +primitive changes are authorized. Successful encoding and null conventions +remain unchanged; exceptional ordinary Sender cleanup is intentionally repaired. + +This replaces the original no-buffer-edit scope after evidence showed it would +leave the new feature unsafe. Keep the failing test, add VARCHAR/BINARY prefix +failures with exact exception identity, and add ordinary Sender real-server A/C +regression coverage. Do not manufacture OOM/native faults to claim recovery; +arbitrary invalid pointers and fatal process faults remain caller/environment +limits. Other column families' exceptional mutation paths are separate follow-up +work, not silently claimed fixed by this variable-width repair. + +### Final validation and acceptance + +All final gates passed with zero failures, errors or skips: + +- Client focused aggregate, including the shared buffer suite: 109 unit tests + plus 2 packaging checks. +- Client canonical: 394 unit tests plus 2 packaging checks. +- Server focused: 4 real-server tests. +- Server canonical: 325 tests, including the nine pre-existing binary Sender + regressions in addition to the previous schema/compatibility/conversion gates. +- Server repeats: 55 tests each at seeds 1/2, 42/43 and 1234567/7654321. + +The 18 shared cases (4 BINARY, 14 STRING) are byte-identical in both repositories. +Client tests inspect exact target wire bytes, 130-row bitmap/offset boundaries, +all three binary overloads, invalid inputs and duplicates, lifecycle guards, +completed-row retention and failed-row-only definition removal. Mutable inputs +are changed after their setters return, and native input is freed before +encoding; the encoded bytes still match the original values. RuntimeException +and Error prefix failures preserve the original exception object and completed +data, including when UTF-8 capacity grows before the failure. + +Real-server tests obtain live DESCRIBE identities, assert TYPE_BINARY before +sending and compare every stored byte. Separate legacy senders for BINARY and +VARCHAR input preserve legacy inferred-layout rules while checking server +conversion parity. NULL, omission and present empty input remain distinct. +The ordinary Sender VARCHAR and schema-binding BINARY A/failing-B/C regressions +both store exactly A/C after the shared repair. The SF case checks target bytes +before persistence and exact recovered bytes after source memory has been freed. +It does not replace existing generic replay byte-identity evidence. BINARY has +no separate compression encoding; the test checks its unchanged payload in +Gorilla-enabled messages, not a binary compression algorithm. + +Root also ran an independent JDK UTF-8/literal-byte oracle over the shared corpus: +all 18 vectors passed, including an independently checked complete 00..ff +sequence. The diagnostic uses no client or server converter. It is recorded in +`core/target/schema-binary-2.14/CorpusOracle.java`, SHA-256 +`7e769a145326b333c4936875680fd5e27b0f6a443bfa09538e640aa163be740c`. + +Client commands/logs: `core/target/schema-binary-2.14/run-manifest.md`, especially +`logs/focused-final-with-buffer.log`, `logs/canonical-final.log` and +`logs/install-final.log`. Server commands/logs: sibling +`questdb/core/target/schema-binary.2.14-server/run-manifest.md`, with +`logs/focused.log`, `logs/canonical.log` and all three `logs/repeat-*.log`. +Earlier fixture/compiler failures and the decisive pre-repair behavioural +failure remain separately recorded; they are not final acceptance results. + +Final source and artifact SHA-256: + +- Binding: `1b21f16f9312d06f6a3ba741426fd53f16477c0280a2bd01fd196afa1615f94d`. +- Shared buffer: `e6a9451e82058d29094b89472af387c6cb23e986a70a9e24fb32c35d79da7731`. +- Client test: `b5350a7d261611c8b73a343718f20e2ba6d0474ed6d2b21c591da41b46a2dd35`. +- Server test: `ca056571bd8225c217e2975386d18e8eb19e8773d2b6eac67088659ef0108427`. +- Shared corpus: `aba29365c5e148a5bd0eae5f95e8e68dbd441c9715741d1ab4a8d93f5c89813a`. +- Workspace and installed client JAR: + `b9f658d20a0e00a56a6cf287a6d407a849e389d0aa4390890039605bd5fc7394`. + +Root verified final log summaries, the standalone installed artifact on the +server test classpath, explicit final seeds and project-local JVM temp paths. +All runs were serialized; the client was not rebuilt/reinstalled during server +gates, and no local-client profile was used. Runtime: Corretto 25.0.4 and Maven +3.9.11. SOL source review accepted the final binding/buffer/tests/corpus hashes; +the documentation review's fault-coverage wording correction is incorporated. + +The only new executable production changes in this iteration are BINARY target +binding and the authorized string/binary append-position repair. Ordinary Sender +source, column writer, off-heap primitive and server production remain unchanged; +ordinary Sender benefits only from the shared exceptional-cleanup repair. +No array/symbol cleanup rewrite, new buffer owner or measured performance claim +is included. Native copy/allocation fault recovery remains source-reviewed, +not dynamically fault-injected. The five server compatibility boundaries from +D028 remain unresolved and are not activated by this slice. + +Root accepts iteration 2.14. Design revision 29 updates the contract, status and +conversion inventory. Remaining conversion families, inference, row-boundary +adoption and full Sender activation are still required. No unrelated files were +removed and no commits or pushes were made. + +## D030 — Integrate public Sender early; keep completeness as a release gate + +Date: 2026-09-11. Status: accepted for the bounded integration scope; unreleased. + +The user requested integration E2Es, asked root to validate the assumptions, +then authorized implementation. Root and three SOL agents traced current +Sender construction, setters, rollback, batching, schema lookup, negotiation +and the real-server test infrastructure. This changes D022's sequencing, not +the product contract: all conversion families need not precede the first public +Sender E2E. An incomplete automatically activated client remains unreleased. +Unsupported conversions and missing-table/column inference may report explicit +development limitations; no raw-value fallback, feature switch or alternate +Sender is authorized. Existing legacy-mode behavior remains in scope. + +The first driving test uses normal `Sender.fromConfig`, a pre-created UUID +table, valid A, invalid B after another field has been appended, valid C, and +explicit flush. B must fail locally and leave no partial row; SQL must contain +exactly A/C. A complementary public-Sender socket test checks the actual +DESCRIBE, FLAG_SCHEMA, UUID wire representation and pinned table identity. +The current real-server fixture does not capture Sender wire bytes, and SQL +alone cannot distinguish client conversion from the existing server cast. +An absent-confirmation scripted peer proves protocol fallback, not execution +against an older QuestDB release. No actual old-server run is claimed. + +Validated implementation constraints and simplifications: + +- Always request the extension through existing connection factories. The + existing engine/factory requirement latches on first confirmed support, + before any extended bytes; do not add a user option or delay that boundary + until publication. A failed handshake is not legacy confirmation. +- Resolve before the first effective operation of a row, not at `table()` or + flush. Empty buffer allocation is harmless. Use the existing I/O loop and + readiness/lifetime signals; ASYNC must not guess legacy while offline. +- Keep producer-owned row pinning separate from the latest lookup cache. + Replacement connections clear the latest cache, not buffered snapshots. + A cached conversion rejection refreshes once; compare the relevant target + and table incarnation, not metadata-version inequality alone, when deciding + INVALID_VALUE versus SCHEMA_CHANGED. Cancel the whole partial row. +- Reuse owner-backed global-dictionary encoding with `beginSchemaMessage` and + `addSchemaTable`; the component's single-table local-dictionary encoder is + not a replacement for Sender batching. `copySplitMessage` already copies + the schema flag and table body; validate it rather than introducing another + split encoder. Dictionary-only and commit-only frames remain flag-clear. +- `flushPendingRows` publishes data. Schema adoption must retain completed + old-snapshot rows without using a public flush as layout cleanup. Preserve + row/byte accounting, symbol ownership, per-table ordering, reset/close, + split preflight, deferred commits and store-and-forward behavior. Do not + create a general queue framework or transform bytes at send time. +- `atNow()` is a useful first test condition, not a new restricted API + contract. Both row-completion paths must avoid raw writes into schema-bound + buffers; reuse the established timestamp conversion rules when integrating + designated timestamps. No new converter family is justified merely to + broaden this milestone. + +Root owns design/journal and acceptance; SOL owns client implementation and +public socket tests, another SOL owns server E2Es, and an independent SOL +reviews lifecycle, compatibility and complexity. Builds are serialized per +repository, with an explicit installed-client artifact freeze before server +GREEN validation; the initial real-server RED uses the accepted 2.14 artifact. +Use project-local JVM temp directories because shared `/tmp` has only about +311 MiB free. No destructive cleanup or commit is authorized by this increment. + +The performance skill is used for the performance-sensitive Sender path. +This is authorized functional integration, not a production optimization or +hardware-bottleneck claim. Its pre-optimization measurement gate is not claimed +to have passed; do not bundle speculative optimization into this work. Review +added allocation, locking and repeated scans separately from correctness, and +do not present source inspection as a throughput or latency measurement. + +Validation evidence, final scope and review outcomes will be appended here +after the actual test runs. A first passing E2E alone is not iteration acceptance. + +### First failing public-path tests + +The client socket test failed before production changes: malformed UUID text +was accepted by ordinary Sender and no DESCRIBE was sent. This is the intended +missing integration, not a converter failure. Evidence: +`core/target/schema-sender-2.15/logs/focused-red.log` (one test, one failure). + +The real-server test against the frozen 2.14 client also failed on the public +path: after a string UUID input bound legacy VARCHAR, a duplicate typed UUID +setter reported a local wire-type mismatch. Schema mode must instead keep the +target UUID binding and ignore the duplicate before conversion. Evidence: +sibling `questdb/core/target/schema-sender.2.15-server/logs/` +`red-installed-2.14-schema.log`. Superseded fixture compilation/configuration +errors are recorded by the server owner and are not counted as behavioral RED. + +Root rejected a review suggestion to keep a successfully negotiated legacy +producer permanently in legacy mode after a supporting-server reconnect. That +would silently bypass the agreed one-way upgrade. Pending legacy blocks must +keep their old framing, while the next effective row adopts schema mode. +Uniform schema generations can share a schema frame; only mixed legacy/schema +blocks need separate frames. Any rare mixed-family path must size-check all +groups before its first publish and preserve deferred commit and symbol state. + +### First real-server GREEN, before full integration acceptance + +The first public socket test passed after negotiation/string routing was wired. +A deliberately intermediate standalone client artifact was then installed and +frozen for the real-server test, SHA-256 +`8180dc92d99122013ce3cfcc2d5a4c84781e25b1d3aa58ad0871895a8830a0ed`. +`QwpSchemaSenderE2ETest` passed 1/1 against that artifact. It uses normal +`Sender.fromConfig`, pre-created schema, string/UUID inputs into the same UUID +target, target-first duplicate handling, local cancellation of invalid B, +same-table C without another `table()`, `atNow`, explicit flush/ACK and exact +A/C SQL. No manual binding/encoding occurs in the test's write path. + +Server evidence: `core/target/schema-sender.2.15-server/logs/` +`intermediate-first-green-final.log`. The prior run reached its SQL assertion +but incorrectly nested a query memory check inside a live-server context; +the final query uses the fixture's normal outer memory-leak check. The installed +artifact hash was unchanged through both runs. Client work continued without +reinstallation during that gate; the server owner released the artifact only +after stopping Maven. This checkpoint does not validate unfinished setter, +refresh, generation, split or recovery paths and is not iteration acceptance. + +### Rejection recovery clarification + +An implemented input family refreshes once on cached INVALID_VALUE or +UNSUPPORTED_FEATURE, including a column that was absent from the cached schema +but has since been added. Compare table incarnation and the relevant column's +existence/type/parameters. An unchanged target preserves the original reason; +an unrelated version change does not become SCHEMA_CHANGED. An intrinsically +unimplemented input family fails directly. Authorization/availability failures +are not conversion retries. Freshness is per setter: a lookup by an earlier +setter in the row does not make a later setter's rejection freshly validated. +This sharpens the existing error contract without adding setter replay or +retaining original values. A sender-reused lookup-result holder may report +cache/fresh origin without allocating a result per row or using an exception +for an ordinary cache miss. + +### Candidate 2 integration validation + +The second installed candidate, SHA-256 +`78cb193a394dbff4136c45118bc24113fd0698180f4598e943e4f8cb946064e7`, +passed all eight real-server `QwpSchemaSenderE2ETest` cases. These add exact +micro/nano designated timestamps, primitive and Instant inputs, failed timestamp +rollback, cold setter-free `atNow`, relevant versus unrelated DDL with buffered +A, repeated cached MISSING, denied lookup recovery, observable auto-flush, +reset discard and close-only publication. The test's typed UUID uses different +low/high limbs, so reversed argument routing cannot pass. Auto-flush is observed +through ACK 0 and SQL before any explicit flush; no explicit flush can mask it. +Cold `atNow` uses a separate fresh Sender. Timestamp tests also assert total +cardinality, not only filtered values. Reset currently covers the active +generation; do not claim its retired-generation case from this test alone. + +Server log: `core/target/schema-sender.2.15-server/logs/candidate-2-eight.log`. +The installed JAR remained byte-identical during the run. Root independently +checked the log and both workspace/installed JAR hashes. Client focused +consolidation passed 74 tests (`logs/focused-current-3.log`); comprehensive +socket tests and canonical regressions are still pending. Exact candidate +source hashes and commands are in both repositories' 2.15 run manifests. +Root verified Maven 3.9.11, Corretto 25.0.4 and project-local JVM temp storage +against the running Maven and Surefire XML, not only the prior manifest. + +The rare mixed legacy/schema batch uses one frame per table block and a +two-pass size-check/publish loop, reusing the existing encoder. Uniform schema +blocks retain combined/split batching. There is no new queue framework or +send-time transformation. The old split-path at-least-once rule remains: +partial publication failures retain all source rows and retries may duplicate +the published prefix. A review request to reset only published groups was +rejected because it would introduce new bookkeeping and a stronger guarantee. +Size rejection must still publish nothing; deferred commit debt must remain +visible to recovery and close. + +### Candidate 2 broad regression and invariant review + +The broader prior-325 plus new-eight server selection ran 333 tests: 10 failures +and 48 errors, not a passing regression suite. The new eight tests passed in +that run and in three seeded repeats. The separate protocol/discovery/identity/ +feedback baseline passed 214 tests. Existing public-Sender tests now negotiate +schema mode: missing inference, unimplemented conversions and deliberately +different schema-mode semantics expose remaining release work. Some old +conversion fixtures also write through the existing `@TestOnly` buffer seam; +those are not evidence about the supported public row API. Preserve the failing +evidence and classify it; do not silently rewrite legacy expectations to claim +a clean release. The exact failures are recorded in the server run directory's +`logs/broad-failure-lines.txt`. + +Root rejected a mock-only proposed fix that changed a target type while keeping +the same `(tableId, metadataVersion)`. The reviewer could identify no coherent +server path producing that response and retracted the finding. The pair is an +immutable encoding identity, relied on by both buffered rows and replay. Adding +target-change adoption for an unchanged identity would conceal a broken +protocol assumption. The narrow accommodation and impossible mock regression +were removed; real version-changing DDL remains covered. + +Independent production review found no confirmed lifecycle or delivery defect +in candidate 2. Root continues to review observable test sensitivity: exact +wire types and asymmetric UUID limbs, both pinned identities across generations, +no manual cancellation masking automatic rollback, and observable reconnect +confirmation instead of timing assumptions. These checks precede final +iteration acceptance; they do not remove the release blockers above. + +The independent follow-up classified all 58 failures: 38 missing conversions +(22 decimal-related), four missing-inference/lookup-versus-argument-validation +cases, seven documented schema-mode semantic differences, four obsolete +direct-buffer `@TestOnly` fixtures, and five old server-error/message assertions +now receiving local typed rejections. The named LONG/designated timestamp +failures belong to the direct-buffer fixtures, not the public timestamp setters. +This diagnostic selection is not an exhaustive count of missing input families. + +Root also rejected waiting through every observed reconnect gap for a sender +that had already confirmed legacy mode. That would unnecessarily stop legacy +store-and-forward buffering during an outage. Only a never-confirmed sender +must wait to select its initial mode. Once support is confirmed it stays +mandatory; before that, confirmed legacy rows remain valid even offline. The +upgrade test must observe the installed supporting connection before asserting +the next row's mode, rather than use a server-start timestamp to change the +production contract. + +The encoder now rejects table counts outside the wire header's unsigned-16-bit +range before writing a header. Sender also rejects more than 65,535 pending +nonempty blocks before publication, rather than truncating a combined header. +This is conservative: separately grouped frames could theoretically carry a +larger logical batch. Supporting that would require a further batching change; +it is not part of this increment and is unrelated to the million-entry schema +cache ceiling. Boundary tests cover zero, 65,535, negative and overflow counts. + +### Final client gate and artifact freeze + +The final focused selection passed 142 tests plus two packaging checks. The +canonical selection passed 410 tests plus two packaging checks, all with zero +failures/errors/skips. This extends the prior 394-test client selection with +15 public-Sender socket tests and the encoder header-range test; it is not a +claim that every repository test was run. Logs: client run directory +`logs/focused-final.log` and `logs/canonical-final.log`. + +The 15 socket cases include exact target UUID bytes, asymmetric limbs, automatic +partial-row rollback, cached missing tables, no-op and duplicate behavior, +legacy VARCHAR bytes with no describe, both pinned identities across an +unrelated schema update, exact cap-split payloads, retired-generation reset, +startup waiting, actual negotiation deadline/retry and interrupt preservation. +The mixed upgrade sequence completes legacy A, starts legacy B while offline, +observes the installed supporting connection via server PING/client PONG, +finishes B unchanged, then buffers schema C. Flush emits exact legacy A/B +followed by schema C with the correct deferred/final commit flags. No sleep, +reflection or production testing bypass establishes the transition. + +Several old client fixtures used an unconnected Sender with injected connection +state. They now connect to a real non-confirming socket peer through public +construction, retaining the original cap/autoflush/input assertions. Existing +unrelated testing seams were not removed and no new ones were added. The mock +server's additions are ordinary wire PING/PONG and negotiated schema replies. + +The final standalone install succeeded with main JAR SHA-256 +`b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`; +the tests JAR is +`48e9665b4bc5b05f4134899c6e301d6e834a067152dfac88b2d4c4f527e786f8`. +Root independently verified workspace/installed main JAR equality, final +source/test hashes, the 410+2 result and Surefire's Corretto 25.0.4/project-local +temp settings. Exact commands and hashes are in +`core/target/schema-sender-2.15/run-manifest.md`. The artifact is frozen for final +server validation; local installation is not publication or release. + +Separate source reviews found no confirmed production blocker. Root reviewed +test sensitivity independently of the test author. Candidate-2 executable +Sender behavior was retained; the later functional production delta is the +encoder count guard. First-use gating, generation replacement and mixed-family +batching reuse the existing I/O loop, table buffers and encoder. The performance +skill guided allocation/ownership scrutiny and avoidance of speculative +optimization, not measured throughput claims. Lookup/cache overhead has not +been benchmarked. The lookup deadline remains fixed at 1000 ms; the proposed +configuration key is not implemented in this slice. + +### Final server gate, acceptance and remaining work + +Against the frozen final standalone JAR, the eight public-Sender E2Es plus +214 protocol/discovery/identity/feedback/transport tests passed 222/222 with no +failures/errors/skips. Each of three public-Sender repeats also passed 8/8, +with seeds `1/2`, `42/43` and `1234567/7654321`. These are repeated executions +of the same eight tests, not 24 additional scenarios. Server evidence is in +`../questdb/core/target/schema-sender.2.15-server/run-manifest.md`, +`logs/final-222.log` and `logs/final-repeat-*.log` within that run directory. + +The server E2E source hash is +`6e3c29d854e7986170224a9ef8e4cb80ea3e440d3e847ed5d2c8712775501a64`. +Before/after client artifact hashes match. Root independently checked the +222-test result, the final eight-test XML, its standalone installed-client +classpath and absolute project-local temp path. No `-Plocal-client` substitution +was used. Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; +server HEAD remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with +preserved working-tree changes. Final diff whitespace checks passed in both +repositories. No commits or pushes were made. + +Root accepts iteration 2.15 for its stated scope: ordinary Sender automatically +uses the existing schema machinery, fails unsupported writes explicitly, keeps +row/block ownership and delivery semantics, and has working public-path E2E +and exact-wire coverage. Separate SOL reviews found no unresolved blocker in +that scope. This acceptance does not turn the broad 333-test diagnostic into +a passing regression suite or authorize release of the incomplete client. + +The remaining release work is still explicit: + +- Implement the audited conversion families and missing-table/column inference. + The broad failures are useful evidence of these gaps, not tests to disable. +- Separate old-wire compatibility expectations from approved schema-mode + differences in the final regression matrix. Actual old-server distribution + compatibility was not exercised by this increment's scripted peer. +- Extend the ordinary Sender E2E through file-backed SF/process restart; the + existing component/engine recovery tests were retained, but the new public + Sender tests do not claim that additional scenario. +- Finish proposed configuration wiring and validate performance before making + latency or throughput claims. No new conversion framework, user opt-out or + send-time transformation is implied by any of this work. + +Reflection: the integration exposed useful contract and fixture errors sooner +than another converter-only increment would have. The most valuable +simplifications were keeping one connection owner, one buffer owner and the +existing encoder/split path, plus refusing mock-driven changes to immutable +schema identities or legacy outage behavior. Continue with bounded conversion +families exercised through this now-working public path; do not return to a +separate final integration phase. + +## D031 — Public Sender creates and recovers its own backlog (iteration 2.16) + +Date: 2026-09-11. Status: accepted for the bounded public-Sender process-restart +test scope; the complete feature remains unreleased. + +After accepting 2.15, the user asked for next-step reasoning and approved the +recommendation to validate public-Sender restart before another conversion +family. Existing recovery tests construct frames and append them directly to +`CursorSendEngine`, then use Sender to recover. That proves the replay component, +not the complete owner-level path from public setters through file-backed +publication and a producer-process crash. Prioritization follows that risk, +not the number of broad-suite failures attributed to DECIMAL. + +The bounded implementation is a real-server E2E, using the unchanged accepted +2.15 standalone client JAR: +`b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`. +No converter, protocol, recovery API or production hook is planned. A confirmed +production defect must be reviewed before a narrowly necessary fix is made. + +Acceptance sequence: + +1. A child JVM creates normal `Sender.fromConfig` with `sf_dir` and `sender_id`. + Public writes produce valid UUID/SYMBOL A, partially invalid B, then valid C. + B fails locally and is not manually cancelled by the test. +2. A test-only network gate forwards WebSocket upgrade and schema discovery, + but captures and withholds table-bearing QWP data. Wait for both the child's + explicit post-`flushAndGetSequence` publication marker and the captured data. + Verify SQL is empty before stopping the process. +3. Force-stop the child without calling Sender.close, require a bounded process + exit, and close/join its old sockets and relay threads. Start a fresh Sender + with the same public directory/identity configuration. +4. Drain to the real server, compare captured unmasked QWP data before/after + restart, and assert exact A/C values, no B, symbol integrity and ACK progress. + Reopen after acknowledged recovery to check that no rows replay again. +5. Extend the same fixture to A and C encoded under different schema identities, + using the already-tested UUID-to-VARCHAR DDL/whole-row retry sequence. + +Source-backed limits and simplifications: + +- `flushAndGetSequence` returns after producer-owned `appendBlocking` publishes + the mmap-backed frame; it does not wait for a server ACK or imply fsync. + The current MEMORY durability policy survives a JVM crash through the OS page + cache. This is a process-crash test, not a host crash or power-loss test. + Reserved FLUSH/APPEND durability settings remain unsupported and unchanged. +- Exact A/C cardinality is valid because the gate proves zero initial delivery. + This does not strengthen the general at-least-once retry contract. No dedup + table should hide an accidental first delivery or duplicate replay here. +- Compare decoded QWP payloads, not WebSocket masks. Permit extra dictionary + catch-up/control frames during recovery. Test the original table-bearing bytes + and pinned identities; never reconstruct persisted files or inspect private + ACK/segment bookkeeping. +- UUID plus SYMBOL exercises symbol replay integrity. It is not proof that a + dictionary side file alone can reconstruct references after all introducing + frames have been trimmed; that stronger scenario is outside the initial test. +- Reuse the real server fixture and public protocol constants. A small gate is + justified because the existing transparent ACK tee cannot selectively pass + discovery and withhold data. Do not build a general proxy/process framework. + All barriers, process joins and socket/thread cleanup must be bounded. + +Root owns architecture, source validation, design/journal and acceptance. SOL +owns the new server test and serialized server Maven runs; a separate SOL +performs adversarial source/test review. The client artifact stays frozen. +Root verified Maven 3.9.11, Corretto 25.0.4 and artifact hashes. Shared `/tmp` +still has about 311 MiB free: parent and child JVMs must use explicit project-local +temp directories under the 2.16 run directory. Preserve both dirty worktrees; +no commits, pushes or broad cleanup are authorized. + +### First public process-restart checkpoint + +The first one-generation test passed 1/1 against the frozen 2.15 artifact: +server run log `logs/focused-fourth.log`. A child JVM used ordinary Sender to +create file-backed UUID/SYMBOL A/C rows after rejecting partial B. The parent +observed public local publication and a withheld schema data frame, verified +SQL count zero, forcibly stopped the child with nonzero exit, and reopened the +same configuration through a new Sender. ACK progress and exact A/C SQL passed. +At this early checkpoint, exact replay-byte comparison, post-ACK reopen and +the two-generation extension remained pending; it was not full 2.16 acceptance. + +The first three runs exposed fixture errors, not a production recovery defect: +an incorrect hard-coded control flag withheld DESCRIBE, the replacement schema +flag referenced the wrong constants class, and `auto_flush_rows=0` is invalid +configuration syntax. Logs `focused-initial.log`, `focused-second.log` and +`focused-third.log` retain these failures. The final fixture uses public +protocol constants and the already-working 2.15 auto-flush settings. + +Independent review also required explicit child temp storage, one tracked +stdout reader instead of competing/dropping readers, controlled complete masked +WebSocket messages, bounded process exit and relay/reply-thread cleanup. These +are real subprocess/network observations, not injected Sender state. No +production change was required for the first passing checkpoint. + +### Final implementation, review and acceptance + +`QwpSchemaSenderRecoveryE2ETest` now contains two real-server cases. The first +recovers UUID/SYMBOL A/C after local `INVALID_VALUE` rejects partial B, then +reopens the acknowledged slot and verifies no rows replay again. The second +holds A while the server drops UUID `id` and adds VARCHAR `id`; B encounters +`SCHEMA_CHANGED` and is cancelled, then C uses the new schema. The public server +wire cursor verifies two one-row blocks with the same table identity, different +metadata versions and native UUID/VARCHAR types. Both cases require local +publication, captured table-bearing data, SQL count zero before force-stop, +nonzero child exit, acknowledged recovery, identical unmasked QWP payloads and +exact final A/C values. No reflection, private persistence inspection or +manually appended replay frames were introduced. + +The independent SOL review and root review found fixture defects worth fixing +before acceptance: + +- Joining the relay while holding its socket-registration monitor could stall + cleanup. Shutdown marks/closes sockets under the monitor, then joins outside. +- An extra or malformed frame arriving after the initial capture assertion could + set a relay failure that was never checked. Health is now checked after both + relay threads terminate, so a rejected extra frame cannot hide a false pass. +- Exceptional cleanup must explicitly verify the forced child's termination; + interrupted relay cleanup must fail, not silently skip its checks. + +All were corrected without production changes. Final source SHA-256: +`9183f38aebccfac35bd2658aee2ad861d7dcec19adb8bff571ca3ea04356488d`. +Independent adversarial review accepted that source, and root independently +checked the assertions, final logs, artifact hash and whitespace checks. + +Final evidence is under server +`core/target/schema-sender-recovery.2.16-server/`; `run-manifest.md` records +commands, runtime, artifact provenance and superseded development runs: + +- `logs/final-230-rerun.log`: 230 tests, zero failures/errors/skips. This combines + the accepted 2.15 server selection of 222 tests, the two new public recovery + cases and six existing timestamp/text/string-numeric/floating/boolean/binary + component replay methods. +- `logs/repeat-1-2-final.log`, `logs/repeat-42-43-final.log` and + `logs/repeat-1234567-7654321-final.log`: both new cases passed each seed pair. + Earlier green logs predate the final cleanup corrections and are checkpoints, + not the final acceptance evidence. +- The workspace and installed standalone client JAR still have SHA-256 + `b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`. + The earlier 410 client tests and two packaging checks were not rerun in this + test-only increment; they remain 2.15 evidence, not new 2.16 results. +- Maven 3.9.11 / Corretto 25.0.4; parent and child JVMs use project-local temp + storage. Server Maven runs were serialized. Both worktrees pass + `git diff --check`; no producer or Maven processes survived the final runs. + No commits or pushes were made. + +Reflection: the existing production ownership and replay paths survived the +real process boundary without another recovery layer. The added complexity is +confined to a bounded test-local child process and data gate. Keep this public +path as the integration baseline for future conversion families instead of +building another parallel Sender or send-time conversion path. + +Acceptance does not erase D030's 58 classified broad-suite failures or prove +actual-old-server distribution compatibility. Conversion/inference coverage, +configuration wiring and measured performance remain release work. This test +also deliberately leaves power loss and dictionary-side-file-only recovery +after introducing frames are trimmed outside its claim. + +## D032 — Resolve failures without losing the compatibility oracle (iteration 2.17) + +Date: 2026-09-11. Status: first bounded legacy-comparison repair accepted; +the remaining 56 broad-suite failures are unresolved. + +The user authorized exploring and resolving the failures after 2.16. Root +assigned SOL a frozen-client broad rerun, another SOL independent prioritization +and compatibility review, and a third SOL the next functional inference design. +Root owns the source cross-checks, scope, journal and acceptance. The installed +2.15 standalone client remains frozen at SHA-256 +`b269286af96ba34d8b051686b70c515f880b04572e8dc537d2558960617b2531`. + +### Reproduction and corrected classification + +The original 333-test selection ran again. The first result was 12 failures +and 48 errors, but the apparent increase from 58 to 60 was not two new schema +defects. The invalid floating and string numeric corpus tests already send +unflagged legacy frames; their new INTERNAL_ERROR responses contain +`No space left [size=1048576]`. Root rejected the initial classification of +those two as schema/legacy semantic differences after reading their actual +write path and the server error immediately before each failed assertion. + +Evidence: server `core/target/schema-failures.2.17-server/logs/baseline.log` +and `failure-summary.txt`. The working classification is: + +| Group | Baseline count | Resolution rule | +| --- | ---: | --- | +| Missing conversions | 38 | Implement bounded families with source, wire and SQL parity checks | +| Missing inference / validation order | 4 | Restore confirmed-missing inference, preserving lookup failures and server policy | +| Approved schema/legacy differences | 7 | Keep distinct, explicit tests of both contracts | +| Old direct-buffer / wrong-frame fixtures | 4 | Use real legacy wire tests; do not pretend they are public Sender writes | +| Old error shape/message expectations | 5 | Assert local typed failure and row outcome, retaining server rejection coverage | +| Disk-space failures in raw invalid corpora | 2 | Correct runtime storage; do not loosen expected rejection status | + +The first implementation group repairs the Boolean, timestamp, floating-numeric +and string-numeric legacy comparisons, including false comparisons that were +passing. Two SOL authors own disjoint pairs of server test files; the third +reviews all four and root independently checks load-bearing findings. Keep +the schema binding, rollback and public Sender recovery branches unchanged. +Use existing non-requesting WebSocket clients, source-native table buffers and +the existing unflagged encoder. No alternate Sender, schema-disable switch, +production hook, reflection or general test framework is authorized by this +group. Server Maven runs remain serialized. + +### Runtime and review corrections + +Maven `-Djava.io.tmpdir` alone produced a reassuring Surefire XML property but +did not establish the JVM's actual temporary-file root. Root inspected the +running JDK's `File.TempDirectory` and `TempFileHelper` bytecode: both use +`StaticProperty.javaIoTmpDir()`, which retains the startup value. Set the +absolute project-local directory in the fork's actual `argLine`, not only a +later system property. Shared `/tmp` has only about 311 MiB available; the +project filesystem has about 26 GiB. No unrelated temporary files are deleted. +On the corrected focused run, root verified both the actual Surefire Java +launch command and its open WAL/metadata file descriptors under the explicit +project-local `tmp/focused4-argline/junit.../dbRoot` directory, not merely XML. + +This also corrects D031's overconfident parent-temp provenance claim: its +Surefire XML property did not prove the parent test JVM's actual temporary-file +root. Its child used an explicit JVM argument. The passing 2.16 test outcomes +remain evidence; the stronger storage-location claim does not. + +Review findings must survive source checking. Root rejected a proposed broad +timestamp-corpus repair after verifying that `timestamp-units.tsv` contains +only 26 MICROS and 22 NANOS cases (all four invalid cases are MICROS). The +reviewer had conflated it with the separate broader `timestamp-inputs.tsv` +corpus and retracted the finding. No seven-unit normalization framework is +needed to repair that two-unit legacy-wire test. A separate renamed raw-wire +characterization must not claim to execute public Sender's `Instant` or +larger-unit normalization merely because it injects their known output values. + +### Next functional group: confirmed-missing inference + +After the comparison tests are reliable, prioritize the common auto-create path +over choosing DECIMAL merely because it accounts for 22 failures. The reviewed +minimal mechanism is inference within the existing non-owning binding and +caller-owned table buffer, not synthetic server schemas or a second registry. + +- A MISSING response is authenticated absence, not permission to create a + table: `QwpSchemaControl.snapshot` returns MISSING before table authorization. + Server write-time authorization and auto-create policy remain authoritative. + DENIED, UNAVAILABLE and TOO_LARGE must never become inferred schemas; keep + their existing distinct failure/retry semantics. +- Preserve the coordinator's existing result filter: only KNOWN/MISSING reach + the producer; other responses already throw typed errors before identity + reuse. Root's concern that a non-known `(-1,-1)` response might reuse a missing + binding was disproved by tracing `completeResponse` and `resolve`. Test that + invariant with inferred rows, rather than adding a redundant Sender branch. +- Infer established native targets only for already-supported setter families. + Preserve existing timestamp inference: NANOS selects nano precision; other + primitive units and `Instant` select micros. For a known target, its actual + precision still wins. This does not introduce new conversion families. +- Buffer column definitions pin inferred types and existing rollback removes + failed-only definitions. A narrow inferred-column lookup must suppress a + same-row duplicate before checking its setter type; the existing generic + lookup checks type first. Do not globally change legacy duplicate semantics. +- Missing tables keep schema framing with unknown identity `(-1,-1)`; missing + columns on known tables retain the known identity. ACK metadata starts a new + generation through the existing adoption path, never relabeling buffered bytes. +- Infer an unnamed designated timestamp only for a missing table. A known + table without one must continue to reject explicit `at(...)`; `atNow()` + retains server-assigned time. Test all these cases through public Sender. + +This is a source-reviewed next-step recommendation, not implemented inference. + +### First repair: controlled broad-suite result + +The final compiled four-class focused run passed 25/25 with the corrected +JVM-startup temp setting (`logs/focused4-acceptance-2.log`). The identical broad +333-test selection then reported 9 failures and 47 errors, zero skips +(`logs/broad-acceptance.log`). Its 56 failing method names are exactly the first +run's set minus these four: + +- Boolean schema/legacy omission comparison: repaired legacy producer path. +- Timestamp normalization characterization: repaired raw legacy producer path + and narrowed the method name/claim to what it actually executes. +- Invalid floating corpus and invalid string-numeric corpus: their original + rejection assertions pass after correcting temporary storage; these were + environment failures, not repaired production conversion defects. + +Thus two of the original 58 contract/fixture failures are resolved, not four. +The remaining 56 consist of 38 missing conversions, four inference/order cases, +five remaining schema/legacy expectation differences, four old direct-buffer +fixtures and five error-shape assertions. The accepted floating/string and +other Boolean comparisons were also corrected even though they had been green: +they now observe real legacy source bytes instead of comparing local schema +conversion with itself. + +Stronger raw-wire assertions required respecting the public cursor contract: +reading a FLOAT through `getDouble()` quiets a signaling NaN, so inspect the +original fixed-width payload after validating its count, width and absent null +bitmap. VARCHAR's null value accessor is an empty flyweight; test the structural +null flag separately, then compare all non-null bytes, including present empty +strings. Legacy BOOLEAN omission must use its actual non-nullable source layout +to preserve false, rather than introducing a null bitmap. Existing SQL values +and rejection statuses were retained. Development logs preserve the temporary +directory, cursor API/exception declaration and assertion mistakes; only the +final compiled source is eligible for the remaining regression gates. + +### Final acceptance and next boundary + +All final gates used the same four test sources and the unchanged installed +client JAR. Evidence lives under server +`core/target/schema-failures.2.17-server/`; `run-manifest.md` records exact +commands, runtime and superseded runs: + +- `logs/focused4-acceptance-2.log`: all 25 affected tests passed. +- `logs/broad-acceptance.log`: the unchanged 333-test selection has 9 failures, + 47 errors and zero skips. The exact method-set comparison shows only the + four removals described above and no newly failing methods. +- `logs/final-251.log`: 251 tests passed, zero failures/errors/skips. This + retains the previous protocol, negotiation, feedback and public Sender/crash + recovery selection, including text/binary replay, and runs all four repaired + conversion-comparison classes in full. +- `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 25 affected tests passed each seed pair. + These are repeated executions, not 75 additional scenarios. +- `logs/final-before.sha256` and `logs/final-after.sha256` record the frozen + artifact and test-source identities. Final test source SHA-256 values: + Boolean `d4041c6530bb0b980e9b96657b08e12fbad95da34bf223a1eb6b1cebd9da7a53`; + timestamp `7b897ea71a5d710a3cdaa570631c76319439f4905741469f85af8255f778c1d8`; + floating `fe7cc286490c9de1de49a7fe3247962cb387f8525dc74c4ae6e272f15f874aed`; + string-numeric `68d9fc992b6c07242acb306acc7df1356610ce7b008a3544df0abd6147f0e269`. + +The independent SOL reviewer accepted the final wire/null/NaN checks; root +independently verified those contracts, logs and source hashes. Client tests +were not rerun or repackaged in this server-test-only increment. No production +source, conversion policy, fixture opt-out or testing hook was added. Existing +dirty work was preserved; no commit or push was made. + +Reflection: passing SQL is not an independent conversion oracle when both +writers now run the same client conversion. Inspect the emitted source bytes +and keep client conversion and server conversion as separate tests. Conversely, +an increased failure count is not proof of another feature regression: inspect +the server cause and actual runtime configuration. Preserve these corrections +before moving on to the source-reviewed auto-create inference implementation. +The complete feature is still unreleased; this iteration does not claim to +resolve all failures, prove old-distribution compatibility or measure performance. + +## D033 — Confirmed-missing inference using the existing buffer (iteration 2.18) + +Date: 2026-09-11. Status: accepted for the bounded inference scope below; +the remaining 54 broad-suite failures are unresolved. + +Continue with D032's functional priority: permit automatic table and column +creation for setter families whose schema-mode conversion is already supported. +The user reiterated that simplicity is a requirement: read enough surrounding +code to justify each change, and avoid speculative defensive checks. + +Root rechecked the binding, producer row-boundary adoption, coordinator result +filter and server schema discovery/write paths. The buffer already owns column +definitions and row rollback; inference should use that state. No synthetic +server schema, inferred-column registry, protocol change, opt-out or send-time +conversion is needed. MISSING permits inference, not creation authorization. +The existing server remains responsible for authorization and auto-create policy. + +SOL owns the narrow client implementation and client tests; another SOL owns +public-Sender server tests; a third independently reviews contracts and unnecessary +complexity. Root owns integration, serialized builds, source checks and acceptance. +Preserve earlier dirty work and keep additional conversion families out of scope. +Required observations include source-native inferred bytes, known/unknown schema +identity, duplicate handling, failed-row rollback, timestamp precision, ACK-driven +adoption and rejection by server policy. No release or performance claim follows +from this increment alone. + +### Source-review refinement: inferred type conflicts must refresh + +The generic buffer rejects a later row's different wire type with an ordinary +`LineSenderException`. Reusing that error unchanged in inference would bypass +Sender's existing schema-rejection refresh: a cached MISSING result could keep +rejecting a string setter after another writer creates a compatible VARCHAR +column. Both root and the independent reviewer traced this concrete path. + +Keep the generic legacy lookup unchanged. In the binding's missing-column path, +look up the existing buffer column, ignore an already-written value in the same +row, then report a conflicting inferred type as `UNSUPPORTED_FEATURE`. This +uses the existing one-refresh/whole-row-retry path. Do not catch arbitrary buffer +exceptions, parse their messages or add a second retry loop. The name-only lookup +can expose the existing column's type and size; no result enum, callback or new +state is needed. Test unchanged MISSING, newly created server targets and failed +refresh, with completed bytes retained in every case. + +Root also rejected an impossible draft test fixture: current server SQL only +enables WAL on partitioned tables, which require a designated timestamp, and +QWP rejects writes to non-WAL tables (`SqlParser` and `QwpTudCache`). A real +known table without a designated timestamp can prove local `at(...)` rejection +and recovery to another writable table. It cannot prove successful QWP storage +via `atNow()`. Keep local row-completion behavior and server writability separate; +do not change production or weaken server checks to make the fixture work. + +### Initial runtime and simplicity corrections + +The first focused client run compiled and executed 140 tests: five failures, +zero errors. All five new wire assertions assumed an unknown identity was +encoded as discriminator 1 followed by literal `(-1,-1)`. The existing writer +uses discriminator 0 with no pair bytes; discriminator 1 introduces a known +pair. Correct the test readers, not the encoder. Static review missed this +and its claim of proven byte preservation was withdrawn pending a green run. + +Root also removed a proposed duplicate-designated-timestamp guard and its two +nullable-result branches. Public `at()` already commits the row; repeating a +designated setter within one uncommitted row was only a component fixture +scenario, and the known-schema path did not support that new behavior anyway. +Keep across-row inferred precision conflicts typed and refreshable, without +inventing an additional within-row designated timestamp contract. + +After those corrections, the focused client gate passed 140/140. The first +broader gate ran 414 tests and found four obsolete expectations in the text, +boolean, floating and string-numeric binding suites: each still required an +absent column to throw. The implemented families now correctly infer those +columns. Update only these expectations, retaining their unsupported-pair, +parameterized-type and designated-column rejection checks. Packaging checks +did not run after this failed gate; do not count them as passed. + +The corrected canonical client gate passed 414 tests plus two packaging checks. +The standalone client was then installed and frozen at SHA-256 +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. +The first server run stopped at compilation because the new test imported +`LineSenderServerException` from the wrong package. Only that test import was +corrected; the failed log is retained and no server execution was counted. + +Replay evidence is more specific than a generic assurance but not a new +inferred-Sender crash test: the existing, rerun +`QwpSchemaReplayNetworkTest.testOldPeerSeesNoBytesThenSupportingPeerReplaysExactMixedFrames` +persists manually encoded legacy and UNKNOWN-schema frames through +`CursorSendEngine`, reopens through public Sender, and compares both replayed +payloads byte-for-byte. The new inference tests separately prove exact emitted +UNKNOWN bytes and retention across a producer schema-generation replacement. +Existing public-Sender process-restart tests remain in the server gate; do not +describe these separate observations as one new inferred-Sender crash scenario. + +The first compiled server run passed all six new inference cases; the migrated +existing test failed only because its SQL used the reserved word `column` +unquoted in ORDER BY. Quoting that identifier preserved the expected schema +and data. The corrected focused gate passed all 14 cases with the frozen client. +Those runtime results now establish actual ACK adoption, source-native creation, +timestamp-only micro/nano/Instant/atNow behavior and write-time policy checks. + +One existing server diagnostic remains unchanged: disabled creation of a missing +table returns INTERNAL_ERROR (`failed to create table update details`) and then +uses the normal poison-frame terminal escalation. Create/add-column authorization +denial returns SECURITY_ERROR. The tests assert the current distinctions and no +table/column creation; this client increment does not reclassify server failures. + +### Final acceptance and remaining work + +The implemented families are BOOLEAN, LONG, FLOAT, DOUBLE, text as VARCHAR, +SYMBOL, UUID, BINARY and primitive/Instant timestamps. Known server targets +continue to select conversion; absent targets use existing buffer definitions. +No additional binding registry, producer/I/O state, protocol variant, opt-out +or conversion family was introduced. Relative to the saved 2.17 source, the +binding adds 88/removes 60 lines, the buffer adds a five-line package-private +name lookup, and Sender changes only the now-stale unbound-error comment. +Known-column writes do not use the added inferred-column lookup. This is a +source-level cost observation, not a throughput or allocation benchmark. + +Evidence is recorded in client `core/target/schema-inference-2.18/run-manifest.md` +and server `core/target/schema-inference.2.18-server/run-manifest.md`: + +- Client `logs/focused-2.log`: 140 tests passed. +- Client `logs/canonical-2.log`: 414 tests and two packaging checks passed. +- Client `logs/install-1.log`: standalone install succeeded; main JAR SHA-256 + `102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b` stayed + unchanged throughout all final server runs. +- Server `logs/focused-acceptance.log`: 14 tests passed. +- Server `logs/canonical.log`: 261 tests passed, preserving the prior 251-test + gate and adding six inference cases plus four existing auto-create checks. +- Server `logs/broad.log`: the unchanged 333-test selection has 10 failures, + 44 errors and zero skips. Root independently compared exact method names: + only `QwpSenderE2ETest.testAutoCreateBinaryColumn` and `testUuid` disappeared + from the prior 56; no failing method was added. +- Server `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 16 integration/recovery cases passed + each seed pair. These are repeated executions, not 48 different scenarios. + +The final broad failures classify as 38 missing conversions, five approved +schema/legacy expectation differences, four old direct-buffer/frame fixtures +and seven diagnostic expectations. Two former inference/order cases now reach +their later assertions: binary argument validation expects the old message +capitalization, and the column-mismatch test expects old text after a flush +that does not await ACK. The latter now sees the required MISSING-to-KNOWN +refresh and `SCHEMA_CHANGED`. Do not change production semantics to satisfy +these stale messages. Classifications describe this selected suite, not an +exhaustive count of all release gaps. + +Root verified the canonical server fork's actual startup temp argument and +open database descriptors under the project-local directory (PID 4123391), +not only Surefire XML. SOL independently reviewed the final production path, +wire tests and failure transitions. Source/test review mistakes were corrected +against actual protocol bytes and runtime results before acceptance. Existing +dirty work was preserved; no commit or push was made. + +Reflection: the existing buffer already had the necessary ownership and state. +The useful additional behavior is confined to selecting an inferred type and +reporting a genuine conflict through the existing refresh path. Read both the +public API lifecycle and the actual wire format before adding guards or tests. +Next, repair stale diagnostics while preserving independent legacy rejection +coverage, then continue bounded conversion families. Unsupported inputs, +old-distribution compatibility and the proposed lookup-timeout configuration +remain release work; this increment does not claim complete feature readiness. + +## D034 — Iteration 2.19: repair diagnostics without changing behavior + +Recommendation: fix the seven stale diagnostic expectations before adding +another conversion family. This is server-test-only work. The accepted 2.18 +client JAR stays frozen at SHA-256 +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +The selected tests now encounter schema-directed setters, so old generic error +text and asynchronous error callbacks no longer describe the operation being +tested. This is not a reason to change production behavior or accept either +local or server errors interchangeably. + +Source-backed decisions: + +- Assert `LineSenderSchemaException` and its reason for local invalid values or + unsupported conversions. Check useful table/column/type context without + pinning incidental capitalization. +- Preserve independent legacy BINARY-to-CHAR/STRING/SYMBOL/VARCHAR rejection + coverage. A nonrequesting WebSocket client sends an unflagged frame; parse + and assert its TYPE_BINARY definition and exact opaque bytes before sending. + Assert server SCHEMA_MISMATCH and zero rows, then separately test public + Sender's local rejection and partial-row recovery. `QwpWalAppender` explicitly + rejects TYPE_BINARY in all four target branches. +- Await the first row's ACK in the inferred-LONG mismatch test. In + `CursorWebSocketSendLoop`, schema feedback is applied before the acknowledged + sequence advances. This establishes the known LONG target before rejecting + fractional 3.14; a valid 3.0 then checks recovery. A sleep or an assertion + accepting multiple reasons would hide the actual lifecycle contract. +- Keep the UUID test's exact completed-A/failed-B/successful-C SQL result. + Exercise recovery without manual `cancelRow`; the binary cases additionally + continue without table reselection. + +SOL owns the two narrowly edited server test files and serialized validation; +a separate SOL reviewer checks compatibility, rollback and test independence. +Root checked the setter/flush/ACK ordering and the server rejection branches. +No production code, general test framework, private-state hook or opt-out is +needed. No client rebuild/install is planned, and old server distributions are +not covered by these current-server legacy-frame tests. + +Evidence limitation: the requested pre-edit copies of the two server files +were not captured by the authors. Do not reconstruct them and call them +snapshots. Review uses the observed original methods, existing worktree diff +and D033's recorded test evidence; the final source hashes and exact failure +method comparison will be recorded separately. Existing unrelated dirty work +must remain intact. + +### Final acceptance + +The iteration is accepted for the seven diagnostic tests. Both authors and an +independent SOL reviewer checked the source; root also checked exact wire, +setter rollback and ACK ordering. A stale comment claiming BINARY still passed +unchecked into VARCHAR was removed. Root requested continuation without table +reselection and an empty-flush assertion so the test observes recovery rather +than supplying cleanup itself. + +Server evidence lives in `core/target/schema-diagnostics.2.19-server/`, including +the exact commands in `run-manifest.md` and preserved logs: + +- `logs/focused.log`: seven affected tests passed. +- `logs/canonical.log`: the previous 261-test green selection plus seven + diagnostic tests passed, 268 total; no skips. +- `logs/broad.log`: the unchanged 333-test selection ran with seven failures, + 40 errors and no skips. Root independently compared method names against + D033: the six owned `QwpSenderE2ETest` diagnostics and + `QwpWebSocketSenderReceiverTest.testColumnTypeMismatchThrowsClientSide` are + exactly the seven removals; no method was added to the failing set. +- `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 23 selected diagnostic/integration/ + recovery tests passed each seed pair. These are repeated executions, not + 69 distinct scenarios. + +The remaining 47 classify as 38 missing-conversion cases, five approved +schema/legacy expectation differences and four obsolete direct-buffer/frame +fixtures. This is the selected regression suite, not an exhaustive release +backlog. Source-null text conversion decisions, unsupported input families, +actual older-distribution compatibility and the proposed lookup-timeout +configuration remain separate work. + +Final server source SHA-256 values: + +- `QwpSenderE2ETest.java`: + `4246989e6899c27035a37a3b7a586028a6d001f0704f6151880d3a6c8d132219`. +- `QwpWebSocketSenderReceiverTest.java`: + `f167130d6770f3c1c2b8961ab29cd2a47cc3fb1253d34f6eb4150230d3cc5990`. + +Before/after artifact and test-source hashes match. Root independently verified +the canonical fork's startup `java.io.tmpdir` and open database files under the +project-local directory (PID 4145918). No client rebuild/install or client test +rerun was needed; D033's 414 client tests plus two packaging checks remain +previous-iteration evidence, not new 2.19 executions. Diff checks pass; existing +dirty work was preserved, and no commit or push was made. + +### Next-step reflection: source nulls before text output + +Read-only planning identified a contract decision before LONG-to-STRING/VARCHAR/ +SYMBOL. The server's `putFixedToStringColumn`, `putFixedToVarcharColumn` and +`putFixedToSymbolColumn` check `cursor.isNull()` before decimal formatting. +`QwpFixedWidthColumnCursor.advanceRow` recognizes LONG's sentinel only when +there is no null bitmap; a supplied `Long.MIN_VALUE` marked present by a bitmap +therefore becomes decimal text instead. D013's approved client exception is +limited to numeric targets and does not silently authorize text targets. +Correction from D035's first runtime test: the formatter emits literal text +`"null"`, not decimal minimum. The bitmap-dependent NULL-versus-text contrast +is real; the initially predicted text was wrong. + +An initial SOL suggestion that UUID-to-text avoided this decision was rejected +on source review and retracted. Both public UUID paths always append the two +limbs; neither treats `(Long.MIN_VALUE, Long.MIN_VALUE)` as a no-op. The same +server cursor checks the UUID sentinel only without a bitmap, and both text +appenders otherwise format it as a UUID. It is not a decision-free alternative. + +Recommended next step: characterize these supplied-sentinel text cases through +the public and legacy-wire paths, then explicitly agree that source nulls remain +null before formatting in schema mode. That would be simpler than reproducing +batch-dependent behavior, but it is a proposed compatibility exception, not an +implemented or newly approved contract. No new converter is added in 2.19. + +## D035 — Iteration 2.20: characterize source nulls before text conversion + +The user requested proceeding from D034's next step. Scope: establish runtime +evidence for the five disputed pairs, then recommend an explicit contract. +This is characterization, not conversion activation. No production code, +public API, negotiated mode or server acceptance rule is changed. The 2.18 +client artifact remains frozen at SHA-256 +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +Source review covers public Sender, the non-owning binding, native column +appends, server dispatch, fixed-width cursor null handling and the actual +formatters. LONG-to-STRING/VARCHAR/SYMBOL and UUID-to-STRING/VARCHAR are accepted +legacy QWP conversions. UUID-to-SYMBOL is not included or newly authorized. +The supplied sentinels are effective writes, not no-ops: LONG minimum is +appended as eight bytes, and the UUID pair as two eight-byte limbs. Only the +server cursor's interpretation depends on the presence of a bitmap. + +SOL owns one new server test class and a separate SOL agent owns serialized +validation; an independent SOL reviewer checks the contract and assertions. +The new file's pre-edit state is explicitly absent. Existing production and +fixture files stay unchanged; no opt-out, private-state test hook or general +conversion/test framework is needed. + +Implemented and validated test matrix: + +- Legacy unflagged LONG frames to STRING, VARCHAR and SYMBOL: ordinary signed + values and range boundaries, including MIN+1/MAX, plus supplied MIN without + a bitmap; contrast a frame containing supplied MIN and an omitted companion + row. Assert source type, original bytes, exact bitmap bits, SQL text and + `is null`, not merely the cursor's interpretation. +- Legacy unflagged UUID frames to STRING and VARCHAR: an asymmetric ordinary + UUID, either limb alone equal to MIN and both limbs equal to MIN; contrast + the all-MIN pair without and with an omitted companion row. Assert raw + limb order and canonical text explicitly rather than using the production + formatter to compute expected values. +- Public schema-aware Sender for all five pairs: ordinary and sentinel native + inputs remain unsupported in the current build. Assert typed local rejection, + completed row A, rollback of partial B and successful text row C without + manual cancellation or table reselection. This does not claim that the + proposed converters or proposed null normalization are implemented. + +Recommendation, now runtime-backed but pending explicit contract agreement: +when these five conversions are implemented, normalize the supplied source +sentinels to target SQL NULL before formatting, independently of surrounding +rows. Keep effective-write and duplicate rules; do not reinterpret them as +setter no-ops. All other LONG values use signed decimal text, and all other +UUID limb pairs use canonical lowercase UUID text. This is a deliberate +exception to bitmap-present legacy results, not a claim of exact parity. +Do not extend it to FLOAT/DOUBLE, IPv4, LONG256, dates, timestamps or other +targets by analogy. Public UUID setter documentation currently does not state +this sentinel rule; implementation must document the approved scope. + +### First runtime findings and oracle corrections + +The first focused run selected all three tests: UUID characterization passed, +LONG characterization failed on its first bitmap-present STRING target, and +the public recovery case reached an invalid query-assertion configuration. +The log is preserved at server +`core/target/schema-text-source-null.2.20-server/logs/focused.log`. + +The LONG result was literal text `"null"` with `v is null = false`, not decimal +minimum. The supplied eight-byte MIN and row-0-present bitmap were exactly as +asserted. Root and SOL had traced the appender to `Numbers.append` but had not +read that overload's own sentinel handling. `Numbers.append(sink, long)` calls +the overload with `checkNaN=true`; its MIN branch writes `"null"`. The variant +with `checkNaN=false` would print decimal minimum, but is not called here. +Correct the expected literal and current design table; do not change production +or confuse the string `"null"` with SQL NULL. D034's predicted decimal text is +explicitly corrected above. UUID's formatter has no corresponding special +case and its source/byte/SQL expectations passed. + +The second failure is fixture misuse, not ingestion: `returnsOnce()` rejects +`.timestamp()` configuration. Remove that unnecessary setting while retaining +the exact SQL row values, timestamp strings and ordering. Adding `expectSize()` +would not make that configuration supported. This illustrates why checking +the test helper's actual contract matters as much as checking the formatter. +All subsequent validation must use a newly frozen source hash and separate +logs; no broad or repeat gate ran against the initial failing draft. + +### Final acceptance and next decision + +The corrected characterization is accepted. Source and adversarial SOL review +found no remaining issue after the two corrections; root independently checked +the actual formatter overload, query helper, frozen source and run results. +Automatic flush triggers are pinned using the existing configuration so row A +remains buffered across B's error. Legacy sockets are owned directly by +try-with-resources, including handshake failure; no extra ownership flag is +needed in a helper. + +Final new test SHA-256: +`47f9bf1ac588ed7c2b6ca74235a330a6f8c6e70adf8a90a6ba05ab9fd05bf816` +for server `QwpSchemaTextSourceNullE2ETest.java`. +Server evidence is under `core/target/schema-text-source-null.2.20-server/`: + +- `logs/focused-acceptance.log`: three tests passed, covering all five pairs + through loops; zero failures, errors or skips. +- `logs/canonical.log`: 271 tests passed, the previous 268 plus the three new + characterization tests; zero failures, errors or skips. +- `logs/broad.log`: the original 333-test selection has seven failures, + 40 errors and no skips. Root independently compared exact method names: + the same 47 as D034, with neither additions nor removals. The three new tests + are deliberately outside this unchanged comparison selection. +- `logs/repeat-1-2.log`, `logs/repeat-42-43.log` and + `logs/repeat-1234567-7654321.log`: all 26 selected characterization, + diagnostic, integration and recovery tests passed at each seed pair. + These are repetitions, not 78 distinct scenarios. + +`run-manifest.md` records exact commands, the preserved initial red run and +initial source hash, and corrected before/after source/artifact hashes. +Root independently verified canonical fork PID 4168481's startup temp argument +and open database/WAL files beneath the project-local directory. The new test +source stayed frozen through all accepted runs; the two 2.19 fixture hashes and +workspace/installed client artifact remain unchanged. Client production code +was not changed, rebuilt or retested. Existing dirty work was preserved; diff +and whitespace checks pass. No commit or push was made. + +The evidence establishes the current server's legacy-frame behavior and the +current schema Sender's rejection/rollback. It is not proof of an older server +distribution, implemented normalization, or improved conversion coverage. +Keep the two independent legacy characterization tests when implementing the +converters; migrate only the corresponding current-public-rejection cases to +the approved behavior. Those rejections are a baseline, not a permanent goal. + +The next product decision is explicit: should these reserved LONG/UUID values +always become SQL NULL for the five text/SYMBOL target pairs, instead of +sometimes becoming literal text? Recommendation: yes, with legacy mode and +persisted legacy bytes unchanged. Document this compatibility exception before +implementing it; do not add batch-sensitive or send-time transformation. + +## D036 — Plan the remaining 47 repairs — 2026-09-11 + +The user requested an execution plan for all remaining failures, superseding +the tentative next LONG-to-text implementation. The separate +[repair plan](schema-aware-sender-remaining-failures-plan.md) is the execution +ledger; the main design remains the product contract. No production or test +source was edited, and no Maven/test execution was performed for this plan. + +### Evidence and sequencing + +Root and SOL audited the exact D035 broad-failure list against complete affected +tests, public setters/bindings, server dispatch, and concrete parser/formatter +implementations. The baseline remains 333 tests with seven failures and 40 +errors, not 47 distinct production defects. A mechanical comparison of the +plan's ledger with the preserved list found 47 entries, 47 unique methods, +no missing methods and no extras. + +Recommended order, divided into 16 independently reviewable increments: + +1. Nine test repairs: five approved schema/legacy expectation differences and + four fixtures that mutate Sender-owned buffers. Restore independent raw + legacy frames; do not weaken production behavior or add a schema opt-out. +2. Eight text conversions: LONG, UUID and FLOAT/DOUBLE input families. +3. Four ordinary timestamp input/output conversion cases. +4. Four CHAR, LONG256 and GEOHASH target cases. +5. Twenty-two decimal cases: native decimal first, then LONG, STRING, + FLOAT/DOUBLE and finally decimal text output. + +Every increment must retain a working public Sender real-server test, exact +wire assertions, independent legacy expectations, rollback/recovery coverage, +serialized frozen-artifact regression runs, seeded repeats and independent SOL +review. Count reductions are checkpoints, not permission to delete cases or +weaken assertions. A first fixed setter can expose a later stale assertion; +classify it against the approved contract before changing it. + +The plan uses the frozen 2.20 JAR and run manifest as its starting evidence. +The previous 271-test green gate and three 26-test repeats are historical +accepted results, not new planning-turn runs. Future increments must grow that +gate and compare exact names in the unchanged 333-test diagnostic. + +### Simplicity decisions and review corrections + +- Schema metadata already carries the complete packed server column type. + `QwpSchemaControl` writes it with an empty extension block; the client retains + it. GEOHASH bits and decimal precision/scale can use this existing metadata. + Do not invent new parameter fields, a second schema cache, or a new registry, + and do not relax rejection of unknown extension blocks. +- Decimal conversion belongs at setter time. One review suggestion to preserve + arbitrary input scale in schema-bound wire data and leave rescaling to the + server was challenged and withdrawn: it contradicts early conversion + validation. Convert non-null coefficients to target scale/precision and + compatible wire storage, then reuse existing append storage. Decimal-to-text + is different: it preserves source scale because that scale is visible text. + Cover emitted all-null columns separately from null/no-op setters; do not + infer decimal null behavior from fixed-width cursors. +- CHAR tests must distinguish malformed raw UTF-8 from malformed public Java + text. The public UTF-16 input is replacement-encoded before the server sees + it; a raw-byte parser result is not automatically the public API result. +- FLOAT and `Instant` overload coverage goes beyond the exact failing method + names but belongs with its corresponding input family. These extra tests + are not additional entries in the 47-method count. +- LONG/UUID text-null normalization remains an explicit contract prerequisite + for implementation; this plan does not silently extend it to floating-point, + timestamp or decimal inputs. Keep the independent 2.20 legacy characterization. + +The final independent SOL plan review found no remaining substantive scope, +dependency or ledger issue after those corrections. Root checked the ledger +and the relevant packed-metadata and decimal append/wire paths. Reviews inform +the plan; they do not substitute for runtime acceptance of its future changes. + +### Scope boundary + +All proposed functional changes are QWP client encoding/binding changes. SQL +only creates test tables and checks stored values; no SQL parser, planner or +execution-engine change is proposed. No send-time transformation, raw-type +fallback, reflection or new test-only production API is authorized by this plan. + +Zero remaining baseline failures will not establish full feature readiness. +Unrepresented conversion families, unsafe legacy-path compatibility decisions, +an actual older-server distribution, lookup-timeout configuration and broader +lifecycle/performance evidence remain visible release work in the main design. +No commit or push was made. + +## D037 — Iteration 2.21: separate schema expectations from legacy frames + +Status: accepted. The user approved proceeding with the repair +plan. This increment implements phase 1a only: five test repairs, plus one +public Sender counterpart to preserve integration coverage. No conversion or +other production behavior is changed. + +### Scope and source-backed decisions + +- The three Boolean conversion tests remain public Sender schema tests. They + keep every supplied true/false value and omitted row, but now assert the + target-native missing values and explicit nullness: BYTE/SHORT zero, nullable + numeric and text targets null. Independent raw BOOLEAN comparisons remain + in `QwpSchemaBooleanE2ETest`. One review initially recommended moving all + five tests to raw frames; root rejected that recommendation for the Boolean + cases, and the reviewer corrected it against the plan. +- The two floating legacy diagnostics now use nonrequesting WebSocket clients, + standalone native FLOAT/DOUBLE buffers and unflagged frames. They prove + no-bitmap NaN acceptance versus bitmap-present NaN rejection for both source + types and all four integer targets. Both source widths retain the accepted + predecessor, legacy clamp at positive 2^63, and rejected successor. Tests + assert raw bits, column definitions, bitmap bits, row counts/exhaustion, + precise server NACK context, stored values and empty rejected tables. +- Existing schema floating tests exercise bindings, not the public Sender + boundary path. Converting the two diagnostics solely to raw frames would + remove that integration coverage. The added + `testSchemaFloatingNaNAndPositiveTwoToThe63Rules` checks public FLOAT NaN in + a separately flushed block, DOUBLE NaN with an omitted companion row, and + local `INVALID_VALUE` for both positive-2^63 overloads. Completed row A and + valid row C survive two partial-row failures without manual cancellation or + table reselection. Existing auto-flush controls keep A buffered during errors. +- Root corrected a draft cursor assertion: no-bitmap supplied NaN is a cursor + null even though its raw value bytes exist; bitmap-present supplied NaN is + not. Do not replace the byte assertion with the cursor's interpretation. + Independent review also required the auxiliary label column's exact name + and VARCHAR wire type, not only its text bytes. Both fixes preceded the + first test run. + +The old NaN-only Sender error helper is removed because these cases now assert +server wire rejection directly. Shared Sender terminal-policy helpers remain +unchanged. No schema opt-out, shared test framework, reflection, new TestOnly +API or send-time conversion was added. SQL changes are only test queries that +check values and nullness; no SQL implementation was touched. + +### Frozen evidence and validation + +Server evidence directory: `core/target/schema-expectation-repair.2.21-server/`. +`baseline/` preserves both pre-edit test files; `run-manifest.md` records exact +commands, hashes and outcomes. The unchanged client JAR is +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. +Frozen changed test hashes: + +- `QwpSenderE2ETest.java`: + `9c48a484369d588114c291c8f6b8a2e939380ebb04c314523e00649b8ed9cab8`. +- `QwpWebSocketTypeConversionE2ETest.java`: + `253dcc2a8149a05fa57913868a19051f571d5b365d81ec431bbe08161da1a134`. + +Both independent SOL reviewers accepted the final source. Root independently +checked the diffs and current client/server behavior. The prior frozen 2.20 +broad log supplies the before-fix failures; no new red baseline run is claimed. + +- Focused: 25 tests passed, zero failures/errors/skips. +- Expanded green gate: 277 tests passed, zero failures/errors/skips. +- Original 333-test comparison: three failures and 39 errors, zero skips. + The exact 42-method set is the previous 47 minus the five phase-1a repairs, + with no new failing method. No case was removed or renamed. +- Three seeded repeats: 32 tests passed at each seed pair (`1/2`, `42/43`, + `1234567/7654321`), zero failures/errors/skips. These are repeats, not 96 + distinct tests. The corresponding Surefire properties confirm each seed pair. + +Root verified live canonical fork PID 5929's startup temp path and database +descriptors under the project-local directory. Surefire XML confirms the same +temp path and standalone installed client JAR. No client build or client test +run is claimed for this test-only increment. Both changed test sources and +the workspace/installed client JAR retained their frozen hashes through all +runs. Root and an independent SOL reviewer separately verified the exact +five-method removal with no additions. Diff and whitespace checks pass. +Existing unrelated work was preserved; no commit or push was made. + +Next: phase 1b, the four direct-buffer legacy fixtures. The 38 missing-conversion +cases are unchanged. This test-only repair does not implement additional +conversions, approve new null rules, prove an older server distribution, or +make the feature release-ready. + +## D038 — Iteration 2.22: isolate four legacy wire fixtures + +Status: accepted. The user requested the next increment. Scope +is phase 1b only: the two GEOHASH-to-text and two named LONG-to-designated- +timestamp tests in `QwpWebSocketTypeConversionE2ETest`. Existing 2.21 repairs +and all original stored-value expectations are preserved. + +### Decisions and source validation + +The old fixtures obtained a Sender-owned buffer, appended directly to it and +then called public row completion. That bypassed the producer's binding and +column tracking while row completion could negotiate/install schema state. +Do not accommodate that test-only mixture in production. The repaired tests +own standalone native buffers and send ordinary unflagged frames through a +nonrequesting WebSocket connection. Public Sender conversion behavior is tested +separately; no opt-out or new production testing API is needed. + +Root and SOL checked the concrete server paths, not just type names: + +- Positive GEOHASH precision 5 selects binary text formatting, not base32. + Preserve `[22, null, 31, null, 10]` and `[1, null, 0, null, 21]`, including + the non-null zero and leading zero bits. Explicit original timestamps keep + row order deterministic. The GEOHASH column is emitted last and checked as + exact bytes: bitmap marker 1, bitmap `0x0a`, precision 5, three compact values. + Cursor checks separately verify precision, row nullness/values and exhaustion. +- A named designated `TYPE_LONG` column is not timestamp-unit conversion input. + `QwpWalAppender` applies neither micro-to-nano nor nano-to-micro scaling to + this wire type; its designated fixed-width branch copies literal values. + Tests pin both column definitions, absent bitmaps, 8-byte widths, two values, + raw value/timestamp bytes and rowwise values, then exact micro/nano SQL output. +- The existing public Sender designated tests already cover both precisions, + primitive/Instant input, partial-row rollback and cold `atNow()`. They remain + in the focused/regression/repeat gates. Unlike phase 1a, no missing public + counterpart required a new test method. Native public GEOHASH conversion + remains unimplemented and must not be credited to these legacy fixtures. + +Root found one review correction before execution: the draft ACK parser passed +`true` for `schemaNegotiated` despite using a nonrequesting connection. The +final default overload rejects unexpected schema feedback. Exact table names +and native LONG byte assertions were also added. Both independent SOL reviewers +accepted the refrozen source after these corrections. Helpers remain local to +the test class; class documentation now distinguishes public conversion tests +from raw legacy server tests. No reflection, shared framework, or production +change was introduced. + +### Frozen evidence and validation + +Server evidence directory: `core/target/schema-legacy-fixture-repair.2.22-server/`. +The pre-edit source is preserved in `baseline/`; `run-manifest.md` records exact +commands and results. The changed test source moved from +`253dcc2a8149a05fa57913868a19051f571d5b365d81ec431bbe08161da1a134` to +`726112e1bc4c88befd0f40df088ce5409485f59b170d707829a214debafa260c`. +The prior Sender test remains `9c48a484369d588114c291c8f6b8a2e939380ebb04c314523e00649b8ed9cab8`. +Workspace and installed client JAR remain +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +- Focused: 37 tests passed, zero failures/errors/skips. +- Expanded green gate: 281 tests passed, zero failures/errors/skips. +- Original 333-test comparison: zero failures, 38 errors, zero skips. Exactly + the four phase-1b methods disappeared from the prior 42; no new failure. +- Three seeded repeats: 36 tests passed at each pair (`1/2`, `42/43`, + `1234567/7654321`), zero failures/errors/skips. These are repetitions, not + 108 distinct scenarios. Each run's Surefire properties confirm the seed pair. + +No client build/test run or new red baseline is claimed. The accepted 2.21 +broad log supplies the original four failures against the preserved source. +Root observed canonical fork PID 22089's startup temp argument and checked its +completed Surefire path/artifact properties. Broad fork PID 24170's startup +argument and live database descriptors independently confirm project-local +storage. No shared `/tmp` database workaround or overlapping Maven run was used. + +The changed test, previous Sender test and workspace/installed client JAR stayed +at their frozen hashes through all runs. Root and an independent SOL reviewer +separately confirmed the exact four-name removal. No method was renamed or +deleted. Diff/whitespace checks pass; unrelated changes were preserved. No +commit or push was made. + +Phase 1 is complete: all nine expectation/fixture repairs are accepted. The +remaining 38 baseline cases require missing conversions; this does not finish +the complete feature-release checklist. Next is LONG-to-text, with the proposed +source-null rule recorded as an explicit contract before implementation. + +## D039 — Iteration 2.23: approve LONG-to-text source nulls + +Status: accepted on 2026-09-11; contract, implementation and validation complete +for phase 2a only. +The user's “makes sense” approves the preceding LONG-only recommendation: +in schema mode, `longColumn` writing to STRING, VARCHAR or SYMBOL converts +`Long.MIN_VALUE` to target SQL NULL, independently of other rows. This is an +effective first write, so subsequent duplicates remain ignored. Other LONG +values use canonical signed decimal text. Legacy encoding, conversions and +replay remain unchanged. UUID and floating-point text-null rules are not +approved by analogy. + +Two SOL source checks confirmed that existing VARCHAR and dictionary-backed +SYMBOL append paths suffice. Both copy mutable formatter input before returning; +reuse of a formatter sink does not require changing dictionary ownership. +Missing-table/column inference must remain native LONG. Resolve target and +duplicate rules before handling the sentinel; unsupported target pairs must +remain unsupported even for a sentinel input. + +Scope is phase 2a only: three baseline failures, with exact target wire, +public Sender, real-server stored-value and recovery tests. Keep the independent +2.20 legacy sentinel/bitmap characterization and all unsupported UUID checks. +No SQL, server production, send-time transformation or new conversion framework +is proposed. Test preparation was delegated to SOL. Production edits awaited the +pre-edit checks at the decision point; the completed work is recorded below. + +### Implementation and review + +The production change is confined to `QwpSchemaBinding`: permit the three text +targets in `longColumn`, retain target/duplicate validation before the existing +MIN null branch, format other values through `Numbers.append(..., false)`, then +use existing VARCHAR or SYMBOL append paths. One lazily allocated 20-character +sink belongs to the non-thread-safe binding. Numeric writes never format or +allocate that sink; new distinct symbols still require the existing owned +dictionary entry. No new cache, registry, writer, protocol or connection state. + +The shared 18-vector corpus has literal expected UTF-8 and SQL values. Tests +cover MIN+1/MAX, signs/zero, MIN-first followed by an otherwise-invalid UUID +duplicate, omission bitmaps, reset, native LONG inference, and local/global +SYMBOL sequences `42 -> 43 -> 42` to detect retained mutable formatter input. +Public Sender socket tests pin the actual described identity and target bytes, +cancel partial B while preserving A/C, and retain separate SYMBOL/VARCHAR +encoding generations after an explicit setter-triggered refresh. The new +real-server test checks all pairs and SQL nullness. Its SF case seeds a local- +dictionary frame into the real log and replays it through a new Sender; it is +not described as public-Sender crash persistence. Existing public crash/global- +dictionary and two-generation recovery tests remain in the server gates. + +Independent SOL source and adversarial reviews accepted this bounded change. +Root preserved these findings rather than adding production workarounds: + +- A cancelled row can leave an unused global symbol entry. The dictionary is + append-only; row rollback does not rewind it. A draft socket oracle incorrectly + expected B's `7` entry to disappear. The corrected oracle requires `[7, MAX]`, + MAX at ID 1, and only A/C rows. No dictionary-rollback mechanism was added. +- The first broad comparison found another stale LONG-to-STRING rejection in + `QwpSchemaRowBufferE2ETest.testNullOmissionAndUnsupportedUsedColumns`. Change + only its unused target declaration from STRING to BINARY: LONG-to-BINARY is + genuinely unsupported, as confirmed by `QwpWalAppender`'s BINARY branch. + The method name, cancellation, omission checks and exact SQL expectations + remain unchanged. Add this method to the green/repeat gates. Do not substitute + UUID-to-STRING, which the next planned increment intends to support. +- Compile-only fixture mistakes (client Unsafe API, response constant types, + server helper visibility/checked exceptions) and a malformed MISSING-schema + response are retained in development logs, not counted as behavioral red + evidence. Source review did not replace compilation. Valid red baselines + then showed five client conversion errors with one native-inference pass, + and all three new server tests failing on the missing conversion. + +The old mixed text-rejection test now retains only the four UUID ordinary/null- +sentinel cases; its independent legacy LONG/UUID byte/bitmap tests are unchanged. +The original three LONG-to-text methods retain their existing SQL expectations. + +### Focused performance evidence + +The hardware-first skill caused a pre-edit baseline check, not an optimization. +The validated study in `core/target/schema-long-text.2.23-client/perf/` is limited +to the existing warmed LONG-to-LONG binding/readback case. Six fixed-work +baseline runs preceded the production edit; twelve interleaved control/candidate +runs followed. Every run completes 500 million rows with the same checksum. +Basic and pipeline-counter median elapsed differences are +0.233% and +0.155%, +below observed variation; instructions/row are effectively unchanged. No +repeatable 5% regression was detected for this case. This is not proof of +sub-percent equality, new text-conversion speed, or end-to-end Sender throughput. +CPU affinity was fixed, but shared-host frequency/boost/thermals were not +controlled. Independent SOL review verified counts, identities and this scope. + +Final server comparison and seeded acceptance results follow. The first broad +run is retained: 333 tests, one stale fixture +failure and 35 missing-conversion errors; it is not the accepted final baseline. + +### Frozen evidence and acceptance + +Client evidence: `core/target/schema-long-text.2.23-client/run-manifest.md`. +Server evidence: `core/target/schema-long-text.2.23-server/run-manifest.md`. +Both include exact commands and distinguish failed development runs from final +gates. No client install or benchmark overlapped server validation. Every test +fork received a pre-created project-local temporary directory through startup +`argLine`; root observed canonical PID 66407's command line and broad PID 68499's +command line/open database descriptors. The canonical descriptor inspection +missed the already-exited process and is not claimed as successful. Saved +Surefire reports confirm temp paths, standalone installed client dependency and +all three seed pairs. + +- Client focused: 6 tests passed. Expanded client gate: 420 tests plus two + packaging checks passed; zero failures/errors/skips. +- Server final focused: 16 tests passed. Expanded green gate: 288 tests passed; + zero failures/errors/skips. Earlier 15/287 runs preceded the additional stale + fixture repair and are retained separately. +- Original unchanged selection: 333 tests, zero failures, 35 errors, zero skips. + Exactly `testLongToStringColumn`, `testLongToSymbolColumn` and + `testLongToVarcharColumn` disappeared from 2.22's 38-name failure set; no added + failure. Root and an independent SOL reviewer compared the sets separately. +- Final seeded repeats: 43 tests passed at each of `1/2`, `42/43` and + `1234567/7654321`, zero failures/errors/skips. These are repetitions, not 129 + distinct scenarios. An earlier 42-test repeat is retained, not substituted + for the expanded final selection. + +Client binding moved from +`cf1415484f32067b9135e235d1a002607d1a99e5dd195c8c7d558aaae111aaca` +to `e24350a9522ceecb64267038410c914dd6b305299c146d5a9c9974c61cbac600`. +Workspace/installed main JAR moved from +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b` +to `0d2d66ce80e7fb20fd3e7468b70ad2d49888b60cb0108694e30f0d713a8ecb49`. +Client/server corpus SHA-256 is +`0db1278342169f2291fa84027c7179a2097549fce1e1eec4aac60b1dab8988c7`. +Final source hashes, including the three changed server test classes, are in +the manifests. Buffer, Sender, original type-conversion/Sender fixtures and +server production remain unchanged through validation. Diff/whitespace checks +pass; unrelated working-tree changes were preserved. No commit or push. + +Phase 2a is complete. Of the original 47 cases, 35 remain; this is not full +feature-release acceptance. Next is UUID-to-text, but its proposed both-limbs- +MIN null rule still needs a separate decision. Do not infer approval for UUID, +floating, timestamp or decimal null behavior from this LONG-only increment. + +## D040 — Iteration 2.24: UUID-to-text with explicit source nulls + +Status: accepted on 2026-09-11 for phase 2b only. The user's "proceed" explicitly +approved the separately proposed UUID rule: for STRING/VARCHAR targets, both limbs equal to +`Long.MIN_VALUE` become SQL NULL; every other pair produces canonical lowercase +UUID text. A single minimum-valued limb remains a value. The null is an +effective first write, so later duplicates are ignored. This deliberately +differs from legacy bitmap-present formatting of the both-MIN pair. Preserve +legacy ingestion/replay and native UUID-to-UUID bytes. UUID-to-SYMBOL remains +unsupported; no floating, timestamp or decimal policy is inferred. + +Scope is phase 2b, two baseline cases. SOL was assigned client implementation/tests, +server E2Es and independent review. Tests must pin literal target bytes and SQL +nullness, limb order, sentinel boundaries, omission/duplicates, row rollback and +encoding snapshot retention. Keep the independent raw legacy characterization. +No server production, SQL implementation, send-time transformation or generic +conversion machinery is planned. + +The hardware-first skill required a pre-edit gate for the changed value-writing +path. A focused existing native UUID-to-UUID baseline and interleaved comparison +were scheduled to check regression risk, not new text conversion speed or +full Sender throughput. The accepted 2.23 binding and JAR were archived under +`core/target/schema-uuid-text.2.24-client/baseline/` before edits. Their hashes are +`e24350a9522ceecb64267038410c914dd6b305299c146d5a9c9974c61cbac600` +and `0d2d66ce80e7fb20fd3e7468b70ad2d49888b60cb0108694e30f0d713a8ecb49`. +At this pre-edit checkpoint, the last accepted baseline had 35 errors. + +Pre-edit source review found two LONG null-first tests whose UUID duplicate was +described as invalid. It becomes supported for STRING/VARCHAR in this increment. +Use genuinely unsupported BINARY-to-text/SYMBOL duplicates instead, retaining +the same null/first-write and rollback assertions. The SYMBOL-to-VARCHAR rebind +test still correctly uses UUID to trigger refresh: its pinned SYMBOL schema +rejects UUID, irrespective of the refreshed target. Keep it unchanged. + +Behavioral red evidence uses the frozen 2.23 production source/JAR. Client +focused selection ran 11 tests with three missing UUID-to-text errors and eight +passes. Server selection ran four tests: all three new conversion E2Es failed +with `UNSUPPORTED_FEATURE`; the narrowed UUID-to-SYMBOL recovery test passed. +Source review caught malformed draft schema-fixture parameter/designated fields +before compilation and corrected them to the real wire protocol. Neither these +draft errors nor the later reset-oracle correction are production findings. +Raw logs are preserved under each repo's `core/target/schema-uuid-text.2.24-*/`. + +### Implementation and client validation + +Only `QwpSchemaBinding` changes in production: target-directed UUID dispatch, +a dedicated lazy 36-character sink and five fixed-width lowercase hex groups. +The native UUID branch returns before text-null normalization. Existing target +validation and duplicate handling run before conversion, and `addString` copies +the scratch contents synchronously. No change to LONG formatting, shared +Numbers, Buffer, Sender or server production. Independent SOL source and +adversarial reviews accepted this scope. + +The expanded red client gate has 12 tests and four expected conversion errors. +The first candidate run exposed a test-oracle defect: all column definitions +precede all payloads, but the rollback reader expected them interleaved. Both +source reviewers confirmed the encoder contract; fix the reader only. The +corrected focused gate passes all 12 tests. The expanded client gate passes +426 tests plus two packaging checks, with zero failures/errors/skips. The new +public Sender test verifies typed UUID rows on both sides of a UUID-to-VARCHAR +schema change: the old block retains native limbs, the new block contains text. +Refresh is setter-triggered; no passive ACK-adoption claim is made. + +Current candidate binding SHA-256 is +`9d49b8476e844aae15227448d103372c18ebfe620cbb2bd4c49e42528d6743ff`. +After successful install, workspace and installed main JAR both hash to +`c06908f19e82b95625c1f3b5661a06a0fe9ced10637defde7d4f845d817c989d`. +The shared 16-vector corpus hashes to +`3278914c60022786037fb09380b717bfde23f327f750b8e33c938e14f0eaba03`; +root independently checked signed limbs against literal UUID text and UTF-8 hex. +Server acceptance and the interleaved performance comparison were pending at +that checkpoint; their completed results follow. + +### Regression checks and fixture corrections + +The pre-edit hardware study was corrected before acceptance: retired-operation +counts alone did not justify a limiting-resource claim, major faults were not a +CPU error metric, and a fixed footprint was not measured saturation. The final +study records actual pipeline fractions, normalized instruction/cycle costs, +fresh cgroup throttle/OOM deltas and qualified kernel-error visibility. Root +reran the validator and reviewed the evidence before authorizing production. +Six baseline comparisons and one separate systemic run preceded edits; the +systemic run started after client red validation finished. + +All twelve interleaved candidate/control measurements completed 500 million +native UUID rows with the same two-limb checksum. Basic instructions per row +changed by -0.186%; neither elapsed nor normalized instruction/cycle costs show +a repeatable 5% regression. The observed lower candidate elapsed/cycle costs +are not a speedup or mechanism claim: shared-host frequency/boost are uncontrolled. +This check covers native UUID-to-UUID only, not the new formatter, allocations, +public Sender or networking. Independent SOL review checked all twelve raw runs. +See `core/target/schema-uuid-text.2.24-client/perf/comparison.md`. + +The first server focused run passed 20 of 21 tests, including all shared vectors, +source-null/legacy checks and text replay. The new rollback test expected an +omitted LONG column to print blank; `CursorPrinter` and the actual output show +`null`. Change only the two expected cells, retaining the same projection, +rows and rollback checks. The failed source and log are archived. These small +fixture defects reinforce that source review does not replace execution; none +justifies a production workaround. + +### Frozen evidence and acceptance + +Client manifest: `core/target/schema-uuid-text.2.24-client/run-manifest.md`. +Server manifest: `core/target/schema-uuid-text.2.24-server/run-manifest.md`. +Both preserve exact commands, identities and unsuccessful development runs. +No Maven runs or client installs overlapped the fixed-work measurements; no +client rebuild overlapped server validation. Saved reports verify startup +project-local temp directories and the installed client dependency. Root also +observed canonical Java PID 121872 and its open database files under the expected +temp path. The focused live-process snapshots missed the short-lived fork and +are not claimed as successful inspections. + +- Client: final focused 12 passed; expanded 426 plus two packaging checks + passed; zero failures/errors/skips. Install succeeded. +- Server: final focused 21 passed; expanded green gate 293 passed; zero + failures/errors/skips. +- Original unchanged selection: 333 tests, zero failures, 33 errors, zero skips. + Only `testUuidToStringColumn` and `testUuidToVarcharColumn` disappeared from + 2.23's 35-name list; no added failure. Root and an independent SOL reviewer + separately checked both directions and the identical selection/fixture hashes. +- Repeats: 48 tests passed at each of `1/2`, `42/43`, and `1234567/7654321`. + Saved UUID test reports confirm each seed pair and temp path. These are + repetitions of 48 cases, not 144 distinct scenarios. + +Final binding and main-JAR hashes remain `9d49b847...6743ff` and +`c06908f1...17c989d` (full hashes above and in the manifests). Final client UUID +test SHA is `c23aa77b7edeff165bcef36ed60bce72425b45507d60e935227cbd53588ce1c6`; +server UUID test SHA is +`4371b0e3aab331755c4101fffde5384ec2966d01f29f8ecba650ea2273dc17ce`. +All frozen production/test/corpus identities remained unchanged through final +gates. Independent SOL cross-reviews accepted client production/tests and server +tests. Raw legacy LONG/UUID characterization is unchanged; the new SF test is +explicitly component-seeded replay, with public crash/two-generation recovery +retained in existing regression tests. Diff/whitespace checks pass. Unrelated +working-tree changes were preserved. No commit or push. + +Phase 2b is complete. Of the original 47 cases, 33 remain; the full conversion +inventory and release checklist are not complete. Next is phase 2c FLOAT/DOUBLE +text output, after separately characterizing and deciding its source-null rule. +Do not infer that policy from LONG or UUID approval. + +## D041 — Iteration 2.25: floating-point text output + +Status: accepted, 2026-09-11. The user requested that work continue one +increment at a time until the remaining conversions are fixed. Start with +phase 2c, then the timestamp, scalar-target and decimal increments in the plan. +The last accepted original selection still has 33 errors. Do not confuse that +finite test list with the wider source-audited conversion inventory or silently +implement the documented unsafe server boundaries. + +Keep production changes sequential, with SOL implementation and independent +review, real-server tests, exact target bytes, unchanged legacy comparisons +and frozen client artifacts. No SQL-engine changes or general conversion +framework. Reuse the hardware-first regression checks for affected existing +paths; no throughput claim is implied. + +The floating text-null rule remains a separate explicit decision. Source +inspection confirms the legacy cursor recognizes NaN as null only without a +bitmap; when a bitmap marks it present, the server formatter emits text `NaN`. +Root recommends deterministic target SQL NULL and has asked the user, without +treating the general continuation request as approval of that choice. SOL is +preparing raw real-server characterization while this decision is pending. + +A second load-bearing assumption failed source inspection: current server +`Numbers.append(double)` delegates to Ryu, while the client retains the older +double formatter. Both expose the same method/default scale. Differential +checks must establish output compatibility before reusing client formatting; +do not change shared legacy formatting to fix this schema-only feature. + +Accepted 2.24 binding, Numbers source and main JAR are archived in +`core/target/schema-floating-text.2.25-client/baseline/` before any production +edit. Binding/JAR hashes remain `9d49b847...6743ff` / `c06908f1...17c989d`. + +The user subsequently explicitly selected SQL NULL for NaN. This approves all +six FLOAT/DOUBLE-to-STRING/VARCHAR/SYMBOL pairs, with effective-write/duplicate +semantics, no legacy change and no inference for other source families. +The new raw real-server characterization passed: one test covering 12 +source/target/bitmap combinations, exact raw bits, null flags, stored text and +SQL nullness. Its source/report are archived under the server's +`core/target/schema-floating-text.2.25-server/` before schema-positive additions. + +The formatter differential found 1,543 mismatches across 150,004 supplied values +(275 DOUBLE, 1,268 widened FLOAT). Examples include `1e23` and the smallest +DOUBLE subnormal. Reusing old client Numbers is therefore not a simplification +that preserves the contract. Prefer an isolated source-faithful schema formatter; +leave global Numbers and legacy text formatting untouched. + +A parallel timestamp audit corrected a stale plan statement: the server first +converts nanoseconds to micros, but its text formatter emits milliseconds, not +microseconds. Root verified the call sites and UTC pattern and corrected phase +3c. This is a source-backed plan correction, not permission to change server +output. Timestamp sentinel/empty-text behavior still needs characterization. + +The pre-edit performance check completed six fixed-work native DOUBLE-to-DOUBLE +runs with identical raw-bit checksums. Root inspected the source attribution, +resource checks and normalized counters and independently ran the study validator. +This is a narrow regression baseline, not floating-to-text throughput evidence. +Production edits were cleared only after this check and the client red tests: +six tests ran, five failed with the expected missing-conversion error; the +native-inference/unsupported-target guard passed. The server red run then had +eight tests, seven expected missing-conversion errors and a passing independent +legacy characterization. Both red XML sets are archived in the 2.25 evidence +directories. Independent SOL source review requested one test strengthening: +compare the SF frame identity with its actual describe reply, not just its types. + +The candidate adds a schema-only formatter and an exact server Ryu port; only +package/class names and provenance differ in the latter. The binding adds lazy +scratch and the three text target arms. Its numeric branches, native inference, +target/duplicate ordering and public Sender code are unchanged. Main client +Numbers is byte-identical to the pre-edit source. Independent SOL review accepted +the frozen code and exact wire fixtures. + +Root reproduced the public-binding/encoder differential: seeds 225202601 and +225202602 generate one million DOUBLE and one million FLOAT bit patterns. +The 999,505 non-NaN DOUBLE and 996,216 non-NaN widened FLOAT results, plus 46 +structured cases, match the actual server formatter byte for byte. Random NaNs +are counted and skipped in this formatter comparison; the shared 48-vector +corpus and public tests separately assert their approved target-null behavior. +`core/target/schema-floating-text-audit/run.sh` records source identities and +the exact build/classpaths; `root-replay.log` records the independent replay. + +The six focused client tests and packaging checks passed. The first 432-test +client regression run exposed a stale numeric-only fixture that still rejected +NaN-to-STRING/SYMBOL. Those two negative cases now use unsupported TIMESTAMP and +BINARY targets, preserving their guards and rollback checks. No production fix +was needed. The rerun passes all 432 tests and two packaging checks. The installed +and archived candidate JAR SHA-256 is +`44fdc2bf48f83db01eed089f915ab451c8490ec8d0aea6fbd638c070142dd43b`. +Interleaved native-path performance and server acceptance gates remain pending. + +Read-only preparation for phase 3a confirms that LONG-to-ordinary timestamp needs +only target dispatch: both target units store the raw supplied count. For MIN, +the legacy no-bitmap null and bitmap-present raw value both store the same +SQL-null sentinel (`QwpWalAppender` ordinary TIMESTAMP branch and +`WalColumnarRowAppender.putIntegerToNumericColumn`). This requires no new +source-null policy exception. Keep designated-name rejection and native LONG +inference; do not route raw LONG through a timestamp unit converter. SOL may +prepare tests while 2.25 is validated, but no next-increment production edits +are permitted before this checkpoint is accepted. + +The first candidate's performance result is not accepted as unchanged: all three +paired basic runs add about 25 instructions per native DOUBLE row (+4.62%), +with roughly 2–4% elapsed cost across the event groups. Although narrowly under +the predeclared 5% rejection line, root requested source attribution before +acceptance. SOL confirmed a C2 inlining cliff: the numeric helper grew from 322 +to 420 bytecodes as its switch range expanded from 2–10 to 2–26. The baseline +inlines hot; the candidate is rejected as too large. This is a repeatable native +cost, not a text-throughput result or mere timing noise. + +Authorize only a small separation of the cold text branch, preserving the +original numeric switch and all validation/null/duplicate ordering. Predict +restored hot inlining and removal of the extra per-row instruction work. Retain +the original candidate and all A/B artifacts; require a new frozen artifact, +complete functional gates and interleaved performance comparison. Do not tune +JVM settings, suppress the evidence or lower the acceptance bar. + +The small split restores the numeric helper to 311 bytecodes; all numeric +arithmetic is unchanged. The first server candidate run reached 29 tests and +found one new expected-output mistake: an omitted UUID prints as an empty cell, +not numeric `null`. Correct only that fixture; the finite/NaN corpus, rebind, +SF and legacy checks passed. The earlier attempt failed test compilation in a +prepared phase-3a fixture (a string case label used as timestamp arithmetic). +That test-only mistake was corrected. Preparation of future increments must +not pollute the current test selection: archive future integration additions +outside the source tree until the preceding checkpoint is accepted. The live +integration class was restored to its exact 2.25 snapshot before the rerun. + +Final acceptance: the corrected helper inlines hot at 311 bytecodes. Twelve +new paired measurements have identical fixed-work checksums, and the first +candidate's extra 25 native DOUBLE instructions per row is gone. Basic cycles +show no regression; pipeline-group elapsed remains 0.73–2.99% higher, qualified +as a limited shared-host/group result rather than a speedup or universal +unchanged-performance claim. Root independently checked the raw comparisons, +inlining evidence and study validator. Both candidate histories are retained. + +The final workspace, installed and archived client JAR SHA-256 is +`1173fe3e8dd3fcd764c3513882e4461e608ff000e8d00d1dac6f91e8386f8b70`; +binding SHA is `bb4955a8c347f9b8c9fb6741f9a3b88e10cc8f471f5a8e78084dd5a31e278f1e`. +The final candidate passes 432 client tests plus two packaging checks; root +replayed the differential successfully after the split. Server gates pass +29 focused tests, 301 regressions and three 56-test repeats (seeds 1/2, 42/43, +1234567/7654321). Saved XML verifies startup tmp and seed properties. The exact +original 333-test selector has zero failures and 30 errors, removing only +DOUBLE-to-STRING/VARCHAR/SYMBOL from the previous 33-name list. No additions. +Root and independent SOL review verified the names, frozen source identities +and typed/null/lifecycle contracts. Server production, legacy Numbers, Buffer, +Sender and the original conversion-test source are unchanged in this increment. + +Evidence and commands: client and server +`core/target/schema-floating-text.2.25-{client,server}/run-manifest.md`, saved +reports, both performance comparisons and the separate differential runner. +No commit/push. Phase 2 of the remaining-failures plan is complete; 30 baseline +cases and the separately tracked wider conversion inventory remain. Phase 3a +has reviewed test preparation only, including two corrected fixture mistakes; +its production change must receive its own red tests, measurements and gates. + +## D042 — Iteration 2.26: LONG to ordinary timestamp + +Status: accepted, 2026-09-11. Continue phase 3a after the accepted 2.25 +checkpoint (30 baseline errors). This is a small extension of the existing +LONG setter, not timestamp parsing or formatting. A known ordinary micro/nano +timestamp column takes the raw LONG count in its own unit; there is no implicit +micro-to-nano multiplication. Supplied MIN remains an effective target-null +write. Source review confirms both legacy bitmap contexts store the same SQL +null, so this adds no source-null policy exception. Named designated timestamp +writes remain rejected, and missing schema still infers LONG. + +The implementation brief permits only the target allowlist and the two existing +target-native addLong switch arms. No timestamp driver, protocol field, new +buffer representation or conversion framework is needed. Root and SOL read +the producer path, target-column guards, wire mapping and ordinary timestamp +branches in QwpWalAppender/WalColumnarRowAppender before this brief. + +Prepared tests use a shared 12-vector corpus, exact target-unit bytes, null and +omission bitmaps, invalid duplicates, automatic A/B/C recovery and pinned native +LONG/timestamp generations. Real-server assertions inspect raw stored timestamps +and SQL nullness in both units; independent unflagged frames characterize MIN +with/without a bitmap. Preparation review caught and fixed two fixture mistakes: +MISSING replies carry no identity pair, and a textual corpus label is not an +epoch count. No production implementation has been authorized yet. The native +LONG baseline is being measured against the accepted 1173fe3e... JAR before +red tests and production changes; no concurrent Maven is permitted. + +Root read and independently validated the fresh native LONG baseline study. +Client red: five tests, four expected unsupported-conversion errors; the +native-inference/designated guard passed. Server red: four tests, three expected +unsupported-conversion errors; independent legacy MIN characterization passed. +The authorized production patch adds only two timestamp target checks and two +labels sharing LONG's raw addLong arm (binding SHA 1f1a0396...). + +The first focused client run found four wire-fixture failures. Both source +reviews had missed an existing timestamp encoding discriminator: the default +Gorilla-enabled stream writes raw-encoding byte 0 after the null bitmap even +for zero, one or two values. Root traced QwpColumnWriter and the existing +timestamp corpus test, then requested explicit assertions of that byte in all +new timestamp readers. Native LONG readers stay unchanged. Production did not +change. Record this as a review miss, not a converter bug: exact-wire fixture +reviews must trace the actual per-type writer and cursor, and never substitute +source review for the runtime red/green gate. The corrected fixtures are frozen +and under test; no generalized reader framework is introduced. + +The corrected client run passes all five focused tests plus two packaging +checks, and all 437 canonical tests plus two packaging checks. Final workspace, +installed and archived JAR SHA is +`35f3eaa6c0b663be59df12cc422346756ee120fa60d4ebde15e0d80d7f98cf58`. +Twelve paired native LONG measurements completed identical 500-million-row +work and checksum. Instructions are effectively unchanged, with no repeated +elapsed/cycle regression at the predeclared threshold, major faults or cgroup +throttle/OOM deltas. Root independently validated the final study/comparison. +This establishes no detected LONG-to-LONG regression, not timestamp throughput. + +Server focused (41) and canonical (305) gates pass. The unchanged original +333-test selector has zero failures and 29 errors; exact name comparison removes +only `testLongToTimestampColumn` from the previous 30. Both 1/2 and 42/43 seeded +68-test repeats pass. The final 1234567/7654321 repeat also passes all 68 tests. +Root checked selected XML seed/tmp properties, exact selector equality, +failure-name delta, installed/workspace/archive identities and both frozen +source manifests. Server appenders and original conversion-test source are +unchanged. Evidence and commands are in client/server +`core/target/schema-long-timestamp.2.26-{client,server}/run-manifest.md`. +No commit/push. Phase 3a is complete; 29 baseline errors and the wider inventory +remain, not a release-ready client. + +Final independent SOL acceptance directly verifies all raw server summaries, +the exact one-name delta, selected repeat XML and final artifact/source hashes. +No remaining finding for this slice. + +## D043 — STRING-to-timestamp: fixed parser, narrow timezone adapter + +Status: iteration 2.27 accepted, 2026-09-11. Preparation and implementation +decisions follow chronologically; final acceptance evidence is at the end. +Root and two SOL audits traced the actual QWP STRING path +through `WalColumnarRowAppender.putStringToTimestampColumn` and +`MicrosFormatUtils.parseTimestamp`, then its generated lexical rules, calendar +computation, English locale tokens and timezone rules. Source and public-parser +probes are under `core/target/schema-string-timestamp-preparation/`. + +Reject the initial false choice between a reduced ISO-only grammar and porting +the general compiler. Implement the eleven fixed formats in their real priority, +with exact relevant calendar helpers and a small zone adapter. The server accepts +hour 24 with modulo normalization even with nonzero minutes; fractional widths +are not a generic arbitrary-length ISO fraction. Preserve existing UTF-16-to-UTF-8 +replacement followed by the server's ASCII-byte parsing view. Do not parse Java +Unicode input more broadly than the bytes the server receives. + +The first zone proposal used JDK `ZoneRules.getOffset(Instant)` everywhere. +Root challenged its treatment of separately parsed years after microsecond +arithmetic wraps. The public-server probe disproves it: +`3509450-07-01 00:00:00.000Europe/Berlin` produces +`4929198542690304` micros on the server, while the naive JDK adapter gives +`4929202142690304`, a one-hour difference. Root independently replayed the probe. +The wrapped naive epoch is `4929205742690304`; server future rules still use +parsed year 3509450 rather than the wrapped instant's year. + +Use public immutable JDK rule metadata and preserve the server's split: fixed +and historical offsets through the exact cutoff boundary; recurring transitions +after it, calculated with the separately parsed year and source-faithful +arithmetic. Retain rule order and UTC/STANDARD/WALL adjustments. This requires +only the relevant transition calculation, not its general timezone engine, +Unsafe access, per-year caches or `LocalDateTime` resolution. The English token +index must retain UTC-first insertion, first duplicate wins, longest-first +matching and the source's case behavior. The proposed hybrid still needs broad +differential validation; source review is not proof of complete equivalence. + +Root also found that `Micros.yearMicros` is not uniformly wrapping arithmetic: +negative day counts whose product wraps positive return MIN. Near-MIN day-of-week +subtraction can itself wrap. Preserve both source helpers exactly and cover these +cases, rather than replacing them with Java Time's narrower calendar range. + +Two-digit years use the same class-initialized reference-century calculation as +the server, not a per-value clock lookup. Named-zone meanings depend on JDK +locale/tzdb data. Different process initialization windows or runtime data can +differ; matching-runtime differential tests cannot prove cross-runtime equality. +Record this limitation explicitly. No new protocol fields, version negotiation, +reduced-year contract or SQL-engine change is proposed. + +The 2.27 test preparation caught an incorrect provisional four-digit fraction +expectation before code activation: `.1234` is parsed as `.234000` seconds, +not `.123400`. The `SSz` pattern compiles as two `S` operations; the second is +greedy and overwrites the first millisecond value. This behavior was already +visible in the original public-parser probe and missed in its first summary. +The independent test author caught it when checking the proposed literal corpus. +Root verified the compiler/token explanation. Every provisional corpus row must +be checked against the actual compiled server parser before freezing; unsigned +fraction-width reasoning is insufficient. No production change has begun. + +Preparation progress: root checked 45 deterministic vectors with the actual +compiled server parser after client UTF-8 normalization; zero differences. +Seven reference-century/timezone vectors are separate runtime-dependent cases. +The fixed-width number parser is reusable from client Numbers: signs and internal +underscores are allowed; it is not a digit-only parser. Greedy numeric fields +have different rules and need narrow source copies. Parsed non-null text can +produce raw MIN with a present wire value and SQL NULL; Java-null strings instead +write the null bitmap. Do not import the typed-LONG source-null policy here. + +Root caught and corrected test-only preparation mistakes before activation: +a nano lifecycle expected 1 instead of 1000; a native VARCHAR fixture treated +its int32 offsets as int64 values; the standalone legacy encoder fixture +incorrectly assumed Sender's delta-dictionary prefix. The chosen encoder entry +point matters as much as its per-type writer. The two-character greedy year +`-1` also belongs with reference-century vectors, not fixed 1999 literals. + +The independent hybrid-zone experiment uses public server calendar helpers to +isolate the adapter: all 604 zone IDs and 34,885 deterministic probes agree, +with checksum 6947487320943606609. The deliberately wrong all-Instant control +has 512 mismatches. Root read and independently replayed the diagnostic. This +does not prove the new client parser, locale tokens or copied calendar helpers; +their final binding/encoder differential remains required. + +The fresh native STRING-to-VARCHAR baseline is measured and independently +validated before production edits: six identical-checksum 100-million-row runs, +12 copied bytes per row, fixed 4096-row resets, no new major faults or cgroup +throttle/OOM errors. The measurement is an existing-path regression reference, +not a timestamp conversion benchmark. Test-only activation and red gates follow; +production remains frozen at the accepted 2.26 binding/JAR. + +Red gates complete on that frozen implementation. Client: five tests, one +failure and three errors, all due to missing STRING-to-timestamp conversion; +the native VARCHAR/inference/designated guard passes. Server: six tests, four +missing-conversion errors; independent legacy VARCHAR wire/SQL characterization +and public designated-timestamp rejection/recovery both pass. No fixture or +compile failure remains. Saved red XML/logs are under the 2.27 client/server +target directories. Authoring the narrowly scoped parser may now proceed. + +The corpus grew to 110 literal cases, checked against the actual server parser +after client UTF-8 normalization (zero differences), plus seven runtime inputs +exercised with both units. Extreme micros are not arbitrary representable longs: +the only grammar reaching them has millisecond precision, so wrapping results +are multiples of eight. MAX-7 is reachable; MAX through MAX-6 are not. Tests use +source-proven reachable strings, saturation/wrap boundaries and exact nano +conversion limits rather than inventing impossible MAX fixtures. + +The initial candidate passed the focused client tests but failed broader public +binding/encoder comparison. Root found the missing last fixed-SSS fallback: +`.+12Z`, `.1_2Z` and `.10000` are accepted through fixed integer parsing and +numeric-zone parsing. The fix adds that attempt after the existing greedy +forms, without changing their priority. Expanded ISO-week testing then found +1297-W01 and 1697-W01 differing by one day. The server retains the originally +parsed year's leap flag after adjusting the ISO-week year; the client had +recomputed it. Removing that recomputation preserves source behavior. Both +findings now have shared literal regressions for both timestamp units. + +Validation also caught a diagnostic-fixture error: the scratch encoder reader +assumed Sender's dictionary prefix and a bitmap-length field. Standalone +encodeSchema has neither. Root rejected that run, checked the actual writer, +and required the corrected diagnostic to decode complete frames through EOF. +The failed log remains saved. An independent hour-24 review finding was withdrawn +after reading the prior normalization in the server's compute method; no +production change was made for it. Reviews are inputs, not acceptance evidence. + +Final parser hash `8c8999302508f7758f564cab5df3b12ba1d3d527206faeda88f9e1d9b1a8399d` +and binding hash `71a53f5fe783551819f5277b76767fedfb506b90c85b932ac90fbd8bd33f4d78` +are frozen for final gates. The public binding/encoder differential passes +214,030 cases in each fresh en_US/tr_TR JVM, including all four-digit ISO-week +years, named-zone aliases, transitions, fraction/sign combinations and seeded +inputs. Checksums are 16401770755196548037 and 7304463666663708165 respectively. +Locale-dependent acceptance differs as it does on the server; each run compares +matched runtime data. Focused client validation passes five tests plus two +packaging checks. Canonical, installed-server and native-path performance gates +remain pending; this is not yet an accepted conversion checkpoint. + +Final functional gates now pass 442 client tests plus two packaging checks, +47 focused server tests and 311 server regressions. The unchanged original +333-test selector reports zero failures and 28 errors; exact name comparison +removes only STRING-to-TIMESTAMP from the previous 29. Seeded recovery repeats +are still required before acceptance. + +The final native STRING-to-VARCHAR performance comparison completes all twelve +100-million-row runs with identical byte/checksum verification. Instructions +are effectively unchanged (-0.01%); basic cycles and elapsed consistently rise +about 2–3%. This passes the predeclared 5% rejection threshold but must not be +called unchanged performance. Variable pipeline metrics do not identify a +hardware cause. No new major faults, throttle/OOM events or visible kernel +warnings occurred. Root read the measurements and independently validated the +study. This guard does not measure timestamp-parser or ingestion throughput. + +Final acceptance: all three seeded repeats pass 74 tests each (1/2, 42/43, +1234567/7654321). Root verifies the selected XML seed and project-local tmp +properties after every run, the exact one-name diagnostic delta, mirrored +corpora, frozen source hashes and installed JAR. The final focused gate also +passes after strengthening public error-message context assertions. Accepted +checkpoint: 28 baseline errors remain; the wider conversion inventory and +release gates are still incomplete. Exact commands, failed diagnostics and +frozen identities are in the 2.27 client/server run manifests. + +Final independent SOL audit confirms the same counts, one-name removal, +standalone installed JAR classpath, all three seed/tmp pairs and frozen hashes. +Both run manifests are updated from pending to accepted; no review finding +remains for this slice. + +## D044 — Timestamp-to-text: approved checked normalization + +Status: accepted in iteration 2.28, 2026-09-11. The initial +preparation and approval follow chronologically. Root and SOL source audits confirm both +text targets use the server's millisecond formatter; nano source counts divide +by 1000 toward zero first. Instant's legacy source precision is micros, even +when it contains a finer fractional part. Prefer one schema-only formatter +over a general format compiler or refactoring the now-tested parser. + +Root rejected an initial null-policy finding based on the wrong cursor. +QwpTableBlockCursor always uses QwpTimestampColumnCursor for both timestamp +types, regardless of Gorilla mode. That cursor never infers null from MIN; +only the bitmap decides nullness. A standalone unflagged encoder/server-cursor +probe confirms present MICRO MIN becomes empty text, present NANO MIN becomes +1677-09-21T00:12:43.145Z, and bitmap omissions remain null. Preserve these +semantics; no new timestamp-to-text null-policy decision is needed. The probe +uses the public encoder's Gorilla-advertised, uncompressed-column fallback; +it does not claim a separately configured Gorilla-disabled Sender test. +Its literal output also distinguishes year -1 (`000-1`) from year zero/one +(`0001`); summaries of helpers are not a substitute for full formatter output. + +D021 approved range checking for timestamp targets, not text targets. Legacy +larger-unit and Instant lowering can wrap into unrelated dates. Root recommends +checked conversion into the existing source precision before formatting, while +leaving legacy mode unchanged, and has requested this narrow user decision. +Do not activate the new converter or assume approval while it is pending. +Read-only preparation is under +core/target/schema-timestamp-text.2.28-client/preparation/; it is not an accepted +conversion or a real-server ingestion test. +Root independently replays the probe against the frozen installed 2.27 client +JAR: exit zero and byte-identical literal output. The pending decision remains +timestamp-to-text overflow, not timestamp null handling. + +The user explicitly approved rejection of out-of-range timestamps written to +STRING/VARCHAR. Normalize primitive NANOS by division toward zero; convert +MICROS through DAYS with checked microsecond arithmetic. Convert Instant with +the existing final-value-safe helper (including its negative endpoint handling) +and existing microsecond rounding. Reject invalid units and null Instant as +before; ignore duplicates before conversion or argument validation. Preserve +the source timestamp MIN behavior described above. Legacy code and server/SQL +production code remain outside this increment. + +Implementation starts with public client/server test preparation and a fresh +native-timestamp performance baseline. Production remains frozen at accepted +2.27 until the baseline gate and red tests are checked. Prefer a small separate +schema formatter and reuse timestampTextSink, whose producer-owned ASCII view +can be copied directly into the existing VARCHAR buffer. Do not add another +scratch field, a compiler, send-time transformation or a new replay path without +evidence that the existing structure is insufficient. + +The pre-edit native MICRO baseline passes the hardware-study gate: six fixed +500-million-row runs have identical checksum 3807237185703477635. Basic medians +are 6.956 seconds, 76.790 cycles/row and 550.126 instructions/row. The comparison +will reject a repeatable 5% cycles/instructions increase; this is a native-path +regression guard, not a text-conversion throughput claim. + +Root validates all 63 shared literals independently against checked BigInteger +normalization and the public server formatter (checksum 13924035717609678081). +The test-only red gate reports five client tests (one failure/four unsupported +conversion errors) and four server tests (one failure/two unsupported conversion +errors); the independent legacy ingestion method passes. Native generation +encoding succeeds before the expected text rebind error. Production is now +authorized for this increment. + +Preparation review rejected test defects before implementation: an invalid UUID +setup value, a misplaced catch, and a scratch oracle/reader that incorrectly +treated empty timestamp text as NULL and invented VARCHAR framing fields. +Corrected tests and the oracle read the actual writer and cursor contracts; +failed diagnostics remain saved. Review summaries alone are not acceptance. + +The production review caught an incorrect epoch-day constant (719528 rather +than the server's 719527) and a Java constant forward reference before the +candidate gate. Both were corrected. Target validation now precedes a simple +NANO-versus-micro conversion branch; there is no repeated text-target conversion +decision or new conversion framework. The formatter has one fixed presentation +and reuses the existing producer-owned UTF-8 sink. + +Frozen candidate binding fb36c954 and formatter c99de83d pass root and independent +SOL source review. Client focused validation passes five tests plus two packaging +checks; canonical validation passes 447 plus two. Installed/workspace/archive JAR +SHA256 is bc65098cf79eb28baeeb63ab1617c6479e34ff75302f1fad750161f33139252a. +The public binding/encoder differential checks 160,202 conversions against the +independent server formatter and BigInteger range oracle: zero mismatches, +checksum 4937659523843806437. Server and performance gates remain pending. + +Focused server validation now passes 53 tests, including the new real-Sender +corpus and independent legacy MIN/bitmap ingestion. The original conversion +tests and both server appenders retain their 2.27 hashes. + +The native MICRO A/B guard passes all twelve fixed 500-million-row runs with +the baseline checksum. Basic paired cycles differ by +0.432%, -1.091%, -0.340%; +instructions are effectively equal. No repeatable regression is detected on +this path. This does not measure new timestamp-text or overall ingestion +throughput. Final server regression and repeat gates remain pending. + +The final server regression gate passes 317 tests. The literal original 333-test +selector reports zero failures and 26 errors; name-by-name comparison removes +only testTimestampToStringColumn and testTimestampToVarcharColumn from the prior +28. No new failing method appears. Three seeded recovery repeats are running +before acceptance; the broader conversion inventory remains a release gate. + +Final acceptance: all three seeded repeats pass 80 tests each (1/2, 42/43, +1234567/7654321). Root checks every run's raw result, archived seed/tmp evidence, +unchanged original failure selector and exact two-name removal. Both frozen +source manifests and installed/archive JAR remain consistent. The hardware +study independently validates. Accepted checkpoint: 26 original errors remain; +this is not full conversion or release acceptance. Exact commands and evidence +are in the 2.28 client/server run manifests. + +Final independent SOL acceptance confirms all raw totals, exact failure-name +delta, corpus/source/JAR hashes and all 48 selected repeat XML files. Root also +parses those 48 reports independently: 16 classes and 80 passing tests per run, +with exact seeds, project-local tmp and installed client JAR. Both manifests +now mark phase 3c accepted; no review finding remains for this increment. + +Next-slice preparation is read-only: phase 4a adds CHAR identity and STRING-to-CHAR, +not CHAR-to-text. Preserve first-character truncation rather than introducing +new one-character validation. Public UTF-16 replacement must match actual server +UTF-8 decoding: BMP first characters survive, supplementary-first becomes NUL, +and malformed surrogates become '?'. Reuse short storage; add the missing CHAR +wire-type mapping before activating the native setter. No new sink, formatter, +protocol parameters or conversion framework is needed. Implementation and +new tests are not yet started for this slice. + +## D045 — CHAR identity and STRING-to-CHAR + +Status: iteration 2.29 accepted, 2026-09-11. The user approved the next slice. +Root and SOL reread the server CHAR dispatch, first-character decoder, client +UTF-16 replacement, existing short storage and Sender rollback before edits. +Keep native CHAR as a two-byte code unit. STRING input takes the first BMP +character; trailing text is ignored. Empty, NUL and supplementary-first inputs +produce a present code unit zero. For malformed UTF-16, schema lowering produces +the same '?' as existing client UTF-8 replacement followed by server decoding; +schema mode writes the resulting CHAR directly, not replacement UTF-8 bytes. +Java null remains an effective bitmap-null write. CHAR's stored null sentinel +is zero, so present empty/NUL/supplementary-first inputs also read as SQL NULL. +Malformed raw UTF-8 yields zero on the legacy server path; it is not the same +case as malformed UTF-16 passed to the public string setter. + +Reuse targetColumn's exact-type helper for the native setter, add the missing +CHAR wire mapping, and use a small first-character conversion without allocating +another sink. Do not add CHAR-to-text, SQL/server changes, protocol metadata, +send-time conversion, a new replay path or a conversion framework. + +Public Sender tests provide the pre-edit red gate. A component test that calls +the new binding method is prepared outside the test source tree until that +method exists; do not add a temporary production stub merely to compile a red +test. A fresh native STRING-to-VARCHAR performance baseline precedes production +edits. Root owns serialized Maven and acceptance; SOL owns bounded test, +implementation, independent-review and performance tasks. + +The pre-edit native string baseline passes the validated hardware study: six +100-million-row runs each verify 1.2 billion bytes and checksum +15640344951909530213. Basic medians are 3.340 seconds, 186.271 cycles/row and +916.699 instructions/row. The predeclared guard rejects repeatable 5% normalized +regressions; it makes no claim about new CHAR-conversion throughput. + +Root independently validates all 23 corpus literals against the actual client +UTF-8 sink and public server first-character decoder (checksum +1837560376326485833). Native CHAR expectations come from input code units, not +the expected-result field. Client red reports two tests with one failure and +one missing-conversion error; server red reports four tests with three missing- +conversion errors and the independent legacy test passing. Production is now +authorized. Preparation review corrected circular test inputs, impossible +designated metadata, incorrect error reasons, raw frame/ACK assumptions and a +scratch reader's nonexistent CHAR encoding byte before candidate validation. + +The first focused candidate gate reports five tests with one failure: native +CHAR did not refresh a cached schema on rejection. Initial review incorrectly +attributed this to the mock fixture. Root traced the public setter and found +the missing LineSenderSchemaException catch used by every previously activated +setter. Keep the original failing generation test and add that same existing +rollback-and-refresh handling; do not weaken the test or change cache freshness. +The initial source acceptance is superseded pending this correction and rerun. + +The corrected Sender (4904b4ef) passes the original CHAR-triggered refresh test; +independent review confirms the standard catch is now present. Binding remains +09170988. Focused client validation passes five tests plus two packaging checks. +The frozen candidate JAR is +edb2a9dbae3ba64d0ab3caf07309d4755187eabbfe676ad7a1fc36aa43642c6a. +Root's public binding/encoder differential checks 65,549 STRING inputs against +the actual client UTF-8 sink and public server decoder: zero mismatches, +checksum 13914181573124632075. Performance and remaining regression gates are +still pending; this is not acceptance. + +The existing native STRING-to-VARCHAR hardware guard passes all twelve fixed- +work A/B runs. Paired basic cycles change by +0.659%, +0.795%, +0.114%, with +instructions effectively unchanged and identical output checksums. No +repeatable regression reaches the declared 5% guard. This is not a CHAR +conversion throughput measurement; full functional validation remains pending. + +Client canonical validation passes 452 tests plus two packaging checks. Install +is green; workspace, installed and archived candidate JAR hashes are identical. +Root reruns the hardware-study validator successfully. Server validation now +uses that exact installed artifact, with no overlapping client rebuild. + +The first focused server candidate run reports 59 tests with one test-query +error: the inferred-table assertion used `ts`, but auto-creation names the +designated column `timestamp`. Both writes and ACKs succeeded, as did the two +original CHAR conversion methods. Correct only the test query/header; preserve +the failed run as evidence. No server or SQL production code changes. + +With the inferred-column assertion corrected, all 59 focused server tests pass. +The final server fixture hash is db3cbf8c. The wider regression gate passes +323 tests. The original 333-test selector is unchanged; its diagnostic and +three seeded repeats are still pending. + +The unchanged original 333-test diagnostic now reports zero failures and 24 +errors. Root compares method names against 2.28: only testCharToCharColumn and +testStringToCharColumn are removed, with no additions. Three seeded repeats +are running before final acceptance. + +Final root acceptance: all three seeded repeats pass 86 tests each (1/2, 42/43, +1234567/7654321). Root parses all 51 selected XML reports independently: 17 +classes per run, exact seeds and project-local tmp, standalone installed client +JAR, no failures/errors/skips. Source/corpus/JAR hashes remain frozen. The +accepted checkpoint has 24 original errors remaining, not full release coverage. +Exact commands, raw logs and hashes are in the 2.29 client/server run manifests. + +Read-only next-slice preparation confirms STRING-to-LONG256 can use the existing +32-byte append storage and four local longs; do not add a retained scratch +object or activate native LONG256. Root and SOL checked Numbers.extractLong256, +parseLong256, Long256FromCharSequenceDecoder and putStringToLong256Column: +lowercase 0x, 2–64 hex digits in complete pairs, right-to-left limbs, and rejection +of the four-MIN-limb literal. Java null/omission remains null. These are existing +server rules, not a new policy choice. No implementation has started for 4b. + +Final independent SOL acceptance confirms all raw totals, the exact two-name +failure reduction, all 51 selected repeat reports, frozen source/JAR/corpus +hashes and revision 47's bounded acceptance claims. No review blocker remains. + +## D046 — STRING-to-LONG256 + +Status: iteration 2.30 accepted, 2026-09-11. The user approved proceeding. +Root and SOL reread the server target dispatch, both extractLong256 overloads, +parseLong256, the four-word decoder and the concrete WAL appender. Preserve the +lowercase 0x prefix, complete byte pairs, 2–64 case-insensitive hex digits and +right-to-left limbs. Java null/omission remains null; a non-null literal with +four Long.MIN_VALUE limbs is INVALID_VALUE, as on the server. No policy change. + +Add only the STRING target arm, LONG256 wire mapping and a narrow append helper +using four local longs and existing parseHexLong/addLong256. No retained scratch +object, native LONG256 setter activation, schema state, send-time conversion, +server production or SQL-engine change. Existing stringColumn already has the +typed rollback/refresh handling missing from CHAR's initial activation. + +SOL prepares shared literal vectors, exact public-wire component tests, public +Sender rollback/generation tests and real-server/independent legacy E2Es. Root +owns serialized red, regression, install and seeded repeat gates. A fresh native +STRING-to-VARCHAR baseline and validated hardware study precede production +edits; it is a regression guard, not a LONG256 throughput claim. + +Fresh pre-edit measurements pass the hardware study, independently validated +by root. Six 100-million-row runs verify 1.2 billion bytes and checksum +15640344951909530213 each. Basic median cycles/row 184.538 and instructions/row +916.792; no major faults or new throttle/OOM events. Baseline JAR edb2a9db and +binding 09170988 are archived before production changes. + +Pre-red root review catches two test assumptions: parsing hex as LONG yields +INVALID_VALUE, not UNSUPPORTED_FEATURE; a cached VARCHAR accepts arbitrary +strings and cannot trigger rejection-driven refresh. The generation test will +instead retain a LONG256 block, switch to VARCHAR and use an invalid LONG256 +string to trigger refresh through the same public setter. Keep the existing +cache policy; correct the test sequence, not production. + +Client red confirms six tests with four missing-conversion errors and no +failures. Root independently checks the 23 shared literals against public +server parsing: 10 values, one Java null, 12 invalid, zero mismatches (checksum +12475823559777660557). Review corrected a scratch oracle that had mistaken the +NULL marker for literal invalid text; it now checks actual Java null and exact +bitmap-only encoding and surfaces unexpected parser failures. + +The initial server red has two missing-conversion errors and one legacy fixture +failure. Correct the raw column count to include the unnamed timestamp, and +retain whole-byte padding in the expected hex text. These are test corrections; +the legacy oracle must pass before production edits. The first red log/XML is +retained separately. + +The corrected server red reports three tests with two missing-conversion errors +and the independent legacy test passing, including rejection of odd-length and +four-MIN literals. Root authorizes the Binding-only production change after +both behavioral red gates, the corrected literal oracle and the validated +pre-edit performance study. Existing Sender lifecycle code remains unchanged. + +Independent source review accepts the bounded parser logic. The first focused +build catches the existing error helper's boxed Integer parameter: new calls +must cast the short ColumnType constant to int, as neighboring helpers do. +Correct only those calls and rerun before freezing a candidate artifact. + +The corrected Binding (a57ce9ae) passes six focused client tests plus two +packaging checks; independent source review confirms only the two casts changed. +Frozen candidate JAR: +37169edc56700c7e34ec357d0eb55d8b25886731faf8a65491f7d80fdd4a270a. +Root's public binding/encoder differential completes 40,055 cases with zero +mismatches against actual client UTF-8 encoding and public server parsing. +Performance A/B and broader client/server gates remain pending. + +The native STRING-to-VARCHAR A/B guard passes all twelve fixed-work runs with +matching output checksums. Paired cycles change by +1.271%, +1.152%, +0.055%; +instructions change by about -0.01%. No repeatable change reaches the declared +5% regression threshold. This does not measure STRING-to-LONG256 throughput. +Broader functional validation now resumes with the frozen candidate. + +Client canonical passes 458 tests plus two packaging checks, and install passes. +Workspace, installed and archived JAR hashes match 37169edc. Root independently +reruns the final hardware-study validator successfully. Real-server validation +now uses this exact installed artifact with no overlapping client rebuild. + +All 63 focused server tests pass, including the original STRING-to-LONG256 +method, shared value/null/invalid corpus, whole-row recovery and independent +legacy value/null/omission/rejection checks. The broader server gate is running. + +The wider server regression gate passes 327 tests. The original 333-test +selector is unchanged and now running; final acceptance still requires its +exact failure-name comparison and three seeded recovery repeats. + +The unchanged original 333-test diagnostic reports zero failures and 23 errors. +Root compares names against 2.29: only testStringToLong256Column is removed; +there are no additions. Three seeded recovery runs remain before acceptance. + +Final root acceptance: all three seeded repeats pass 90 tests each (1/2, 42/43, +1234567/7654321). Root independently parses the 54 selected XML reports: 18 +classes per run, exact seeds and project-local temporary storage, standalone +installed client JAR, zero failures/errors/skips. Frozen production/test/corpus +hashes remain unchanged. Client canonical is 458 plus two packaging checks; +server focused is 63, canonical 327, and the original diagnostic remains +333 tests with 23 errors. Revision 48 and the remaining-failures plan record +phase 4b acceptance, not release acceptance. Exact commands, raw logs, hashes +and the selected-report audit are in the 2.30 client/server run manifests. + +Read-only preparation for phase 4c traces the server's string-to-geohash parser, +cursor and target precision checks against the existing client buffer/writer. +Precision is already packed in the full schema type; no protocol parameter is +needed. A null-only column still needs precision initialized without a value +append. Empty text takes the present all-ones path, whereas Java null uses the +bitmap; both read as SQL NULL. Keep this distinction in exact wire tests and +reuse the existing precision field. Native geohash setter activation remains +separate work. No phase 4c implementation has started. + +Final independent SOL acceptance confirms all 54 selected repeat reports, +raw totals, exact seeds/tmp/installed-JAR provenance and frozen fixture/corpus +hashes. Revision 48, the plan and D046 consistently bound acceptance to phase +4b with 23 original errors remaining. Native LONG256 remains unimplemented; +performance evidence is only the native STRING-to-VARCHAR regression guard. +No review blocker remains. + +## D047 — STRING-to-GEOHASH + +Status: iteration 2.31 accepted, 2026-09-11. The user approved continuing. +Scope is phase 4c only: string conversion to existing GEOHASH targets using +their packed schema precision. Native geohash setters and decimals stay separate. +Root delegates client tests/implementation, server E2Es, and performance plus +adversarial review to SOL. Root owns serialized Maven/install gates and records +decisions. No server production or SQL-engine change is planned. + +Revalidation corrects D046's read-only next-slice recommendation. Server VARCHAR +conversion maps empty text to the target's -1 null sentinel at every precision. +But native GEOHASH reads only ceil(bits/8) bytes: without a bitmap, a valid +all-ones value at byte-aligned precisions becomes null; with a bitmap, a packed +empty sentinel can zero-extend to a non-null value in larger target storage. +Root and SOL independently traced QwpGeoHashColumnCursor.advanceRow/readValue, +WalColumnarRowAppender.putStringToGeoHashColumn/putGeoHashColumn and the type's +storage-width calculation. Source presence cannot simply be copied to target +presence. The earlier claim that native present -1 always stores as null was +wrong. Add real-server characterization before production edits. + +Recommended minimal representation: empty text uses addNull, and schema-directed +GEOHASH columns always emit the explicit null bitmap, even when all rows have +values. A private writer argument can distinguish the existing schema/legacy +encoding entrypoints without new retained state or scanning values. Cost is one +bit per row; legacy encoding stays unchanged. This is null framing, not a +send-time value conversion. Initialize precision through the existing column +field, factoring the current addGeoHash validation. Preserve full type flags +and existing timestamp handling; do not globally switch all dispatch to tagOf. + +The hardware-first performance skill requires a fresh pre-edit regression +baseline for the shared string/buffer/writer path. SOL has completed six fixed +100-million-row native STRING-to-VARCHAR runs with matching 1.2-billion-byte +checksums; the validated study and behavioral red gates still precede production +authorization. This guard does not measure GEOHASH conversion throughput. + +Root independently validates the pre-edit study and all 18 shared literal +vectors against actual client UTF-8 plus the current server's public parser: +13 values, two nulls, three invalid, zero mismatches. The initial client red +reports four missing-conversion errors and no failures. Review then corrects +test-only assumptions before the final red: at eight-bit precision `04`, not +`01`, truncates to 1; ordinary encode(table) has no symbol-delta prefix. Expand +permanent tests to all 60 precisions rather than only one 60-bit target. + +Two boundary checks refine the brief without a new abstraction. First, the +existing append benchmark does not call the encoder; extend the pre-edit +measurement to cover the shared writer separately. Second, the low-level +schema encoder accepts manually constructed non-nullable buffers whose bitmap +address is zero. A forced bitmap must be limited to existing nullable storage; +schema bindings always use that storage. Keep manual sentinel-mode encoding +unchanged and test it through the public encoder, avoiding a new allocation or +a null-address copy. + +The final corrected client red reports five missing-conversion errors, no +failures. Server red reports three tests with two missing-conversion errors; +the independent legacy VARCHAR and native GEOHASH characterization passes. +This confirms both packed-width traps at runtime before production changes. +Root also validates the extended two-card performance study: separate append +and encodeSchema baselines, three basic and three pipeline runs each. The +encoding runs verify checksum 114033733800999820 with zero major faults. + +Root authorizes the narrow production change after these gates. The existing +ColumnBuffer.usesNullBitmap() getter expresses the nullable-storage condition +directly; reuse it rather than inspecting pointers or adding a field. Keep +the current full-type switches for all non-geohash targets. Review, focused +green, regressions, A/B, original failure-name delta and repeats remain ahead. + +The first candidate passes five focused client tests plus two packaging checks, +and 7,210 public differential cases across all precisions (6,870 accepted, 340 +rejected, zero mismatches). However, the full 463-test client gate finds three +real regressions: existing flagged STRING/BOOLEAN/LONG schema types are mistaken +for geohashes and produce an untyped zero-precision error. Source-only acceptance +is withdrawn. Preserve these existing tests; recognize a GEOHASH target only +when its precision is 1–60 and its full type exactly matches the canonical +packed type for that precision. Unknown flags/tags must keep the existing typed +UNSUPPORTED_FEATURE path. No generic exception wrapper or new schema state is +needed. Candidate 41722b7b and the failed full-suite evidence are archived as +initial, not accepted; A/B remains on hold until the corrected candidate. + +The server recovery fixture also had an insufficient one-character zero at +eight-bit precision. Change it to `00`, assert the failed row's INVALID_VALUE, +and assert exact A/C/D values and nullness. The red-tested fixture was not copied +before this edit; its archive was reconstructed by reversing exactly those +three edits and verifying the original SHA-256 77ab62ac. Current fixture SHA is +6bd7f662. This is an exact reconstruction, not a pre-edit capture. + +The corrected Binding (b85add96) recognizes only complete canonical GEOHASH +types. The original three regression tests are unchanged; new malformed-type +vectors add zero/61-bit, wrong-tag and unknown-flag coverage. Six focused client +tests plus two packaging checks pass. Independent SOL source review accepts +the correction. Frozen candidate JAR: +c75a826aeb85e9e5f3728d8bd4284c8cd62aab18b8cc3d3c9829e686cc4396c4. +Root reruns the 7,210-case public differential against current server classes: +6,870 accepted, 340 rejected, zero mismatches, checksum 11748646508706949247. +The full client gate is rerunning before install and A/B. + +The corrected full client gate passes 464 tests plus two packaging checks. +Install passes, and workspace/archived/installed JARs all match c75a826a. +The two-path A/B window now runs exclusively, with no overlapping Maven work. + +All 24 counterbalanced A/B runs pass the declared repeatable 5% guard, with +matching fixed-work checksums. The cost is small but measured: native string +append instructions increase about 0.437%, cycles 1.457–1.740%, elapsed +1.461–1.799%; schema encoding instructions increase about 0.472%, cycles +0.131–1.091%, elapsed 0.166–1.148%. Do not describe this as unchanged performance +or infer GEOHASH/ingestion throughput. Root independently validates the final +two-card study. No new throttle/OOM/major-fault or visible kernel-error evidence. + +The final installed candidate passes all 67 focused server tests, including the +original STRING-to-GEOHASH case, all 60 target precisions, null-only following +batches, invalid row recovery and the independent legacy/native characterizations. +The 331-test server regression gate is running before the unchanged original +diagnostic and three seeded repeats. Frozen source/test/corpus hashes still match. + +The wider server gate passes all 331 tests. The unchanged original 333-test +diagnostic is now running; acceptance still requires its exact failure-name +comparison and three seeded recovery repeats. + +The unchanged original 333-test diagnostic reports zero failures and 22 errors. +Root compares raw failing method names against 2.30: only +testStringToGeoHashColumn is removed, with no additions. The remaining original +cases are all decimal conversions. Three seeded recovery repeats are running. + +Read-only phase 5a preparation recommends all three typed decimal inputs as one +coherent increment, with exact target precision/scale and existing wire widths. +Root rejects a second binding-owned Decimal256 scratch: ColumnBuffer already +owns one that can normalize a copy, validate it and append atomically. Preserve +public decimal-null no-op behavior and caller input objects. Internal decimal +word order and wire order differ and need exact independent assertions. +The next slice must also preserve target scale across reset and failed-row +cleanup when an emitted column is later all-null through omission, while legacy +columns retain their first-input-scale-per-batch behavior. The remaining-failures +plan records these tests; no decimal implementation is included in 2.31. + +Final root acceptance: all three seeded repeats pass 94 tests each (1/2, 42/43, +1234567/7654321). Root independently audits all 57 selected XML reports: 19 +classes per run, exact seeds and project-local temporary storage, standalone +installed client JAR and zero failures/errors/skips. Production, server fixture +and corpus hashes remain frozen. A final whitespace-only cleanup in the new +client component test is followed by a fresh six-test plus two-packaging-check +run; the client JAR remains byte-identical. The earlier tested source hash is +preserved in validated-tests-before-formatting.sha256. + +Revision 49 and the plan accept phase 4c, with 22 original decimal errors still +remaining. They do not imply complete conversion coverage or release readiness. +Exact commands, raw logs, hashes and selected-report audit are in the 2.31 +client/server run manifests. No commit, server production or SQL-engine change. + +Final independent SOL acceptance confirms all 57 selected repeat reports, +seeds/tmp/installed-artifact provenance, unchanged final JAR and bounded revision +49 claims. It reruns the hardware-study validator and verifies the measured +cost disclosure, native-setter exclusion, 22 remaining errors and final +whitespace-only test check. No review blocker remains. + +## D048 — Native decimal-to-decimal conversion + +Status: iteration 2.32 and phase 5a accepted, 2026-09-11. Revision 50 records +the bounded scope: Decimal64, Decimal128 and Decimal256 public inputs now reach +DECIMAL8/16/32/64/128/256 targets. These are 18 logical pairs over nine physical +source-width/target-width paths because the three small target types use +DECIMAL64 wire storage. Phases 5b–5e remain; this is neither complete conversion +coverage nor release readiness. + +The implementation reuses the ColumnBuffer-owned Decimal256 scratch. It copies +the caller value, applies exact rescaling, checks declared target precision and +the selected physical storage width, then appends direct target-width limbs. +Precision loss, scale-up overflow, target-precision overflow and storage-width +overflow fail locally without mutating the caller. The target scale survives +buffer reset and failed-row cleanup. Exact component and public-wire assertions +cover high-to-low internal limbs versus low-to-high wire order, row rollback, +completed-row preservation and adoption of a later schema generation. + +Public typed-decimal null/sentinel inputs retain their established no-op behavior. +Confirmed MISSING columns and legacy mode keep source-width inference and the +legacy first-input-scale-per-batch rule; schema-directed known targets do not. +Server characterization confirms decimal cursor nullness is bitmap-only: a set +bitmap bit is null, while raw sentinel limb patterns without a bitmap remain +supplied values entering conversion. This characterization does not activate +decimal text output or the LONG/STRING/FLOAT/DOUBLE decimal inputs. + +The shared client/server corpus has 60 data rows: 45 accepted values and 15 +invalid values, covering all 18 logical pairs. Both copies have SHA-256 +`3819694736f54de4d2f2f396b09ff487667051c6045b34cc6d7579552551d11d`. +Client focused validation passes eight tests plus two packaging checks; canonical +validation passes 472 tests plus two packaging checks. Frozen candidate JAR +SHA-256: `1e814a2f82836b5c9ac04cf7b5674e56f1b939440df486ae6628ebb1b8eebb08`. + +Server focused validation passes seven tests and the canonical gate passes 338. +The unchanged original 333-test diagnostic reports zero failures and 18 errors: +only the four intended decimal-to-decimal methods disappeared from the prior 22, +with no additions. All three seeded repeats pass 101 tests each for seeds 1/2, +42/43 and 1234567/7654321. Root audits 20 archived XML reports per repeat with +zero failures, errors or skips. + +The legacy-path regression guard passes all 12 balanced A/B runs with matching +fixed work and checksums. Paired instructions change by +0.001% to +0.002%; paired +cycles range from -0.47% to +1.99%, below the declared repeatable 5% rejection +threshold. A separate candidate-only composite covers all nine physical width +pairs in six runs of 20 million rows each: 180 million conversions and 3.36 GB +verified per run. Its basic medians are 32.91 ns, 182.03 cycles and 1140.18 +instructions per value. This has no semantically matched schema baseline and is +absolute-cost evidence only; it supports no network, ingestion or general- +throughput claim. One 200-million-row sizing run and an interrupted follow-on +are excluded from the accepted measurements. + +Residual P3: a scale-up overflow has the correct `INVALID_VALUE` reason and row +rollback, but its detail currently says `decimal value cannot be rescaled +exactly` instead of identifying overflow. Keep this as diagnostic wording debt; +do not add another hot-path branch solely to refine the message. + +Final acceptance is limited to phase 5a. The remaining 18 original diagnostic +errors are phase 5b LONG-to-decimal, 5c STRING-to-decimal, 5d floating-to-decimal +and 5e native-decimal-to-text. No commit, server production or SQL-engine change +is included in this documentation update. + +## D049 — LONG-to-decimal conversion + +Status: iteration 2.33 and phase 5b accepted, 2026-09-11. Revision 51 records +the bounded scope: public `longColumn` now reaches every +DECIMAL8/16/32/64/128/256 target in schema mode. Phase 5a remains the decimal +foundation; phases 5c–5e remain. This is neither complete conversion coverage +nor release readiness. + +Root and the SOL contract reviewer read the server's LONG dispatch, decimal +rescale/range helpers, decimal cursor and physical target selection before the +change. The contract is scale-zero signed input followed by exact multiplication +for a positive target scale, declared-precision validation and physical-width +validation. Small target classes use DECIMAL64 wire storage. Duplicate +suppression precedes conversion. Confirmed MISSING keeps inferred LONG. + +`Long.MIN_VALUE` needs an explicit schema-mode rule because legacy fixed-width +sentinel handling changes when another row creates a null bitmap. The accepted +rule is stable source null: for a known decimal target it marks the decimal null +bitmap and first locks the target scale. A raw-frame real-server test preserves +and documents the legacy difference: without a bitmap the sentinel is null; +with a bitmap it is supplied as a value and can either fit or fail by target +precision. No legacy behavior was changed. + +The implementation adds no Sender, protocol or server production state. It +initializes the ColumnBuffer-owned Decimal256 scratch from LONG at scale zero, +then shares phase 5a's exact rescale, precision/storage check and direct-width +append tail. A separate null helper only locks target scale before `addNull()`. +The common numeric-target predicate was deliberately not broadened because that +would silently admit decimal targets for unrelated source families. + +The shared client/server corpus has 39 rows: 19 values, 14 invalid values and +six source-null cases across all six target classes. Both copies have SHA-256 +`d9a47432a14583f8675b0035ab10c3ef9f36e78c9f4aab6a8634958eb5aafcb1`. +An independent arithmetic oracle confirms every outcome, SQL value and unsigned +low-to-high wire limb. Component tests assert target type, bitmap, target scale, +limbs, duplicate behavior, omission, whole-row rollback and reset. Public +Sender E2Es assert exact stored values and A/failed-B/C recovery. + +The first schema-generation fixture was wrong: its proposed new LONG value was +also valid under the old DECIMAL(18,4) schema, so no refresh was justified. +Root replaced it with a value invalid under DECIMAL(18,4) but valid under +DECIMAL(20,5). The corrected test proves that the sealed old DECIMAL64(scale 4) +block remains unchanged while the later block uses DECIMAL128(scale 5). This +was a test-only correction; no production response to the false assumption was +added. A first raw legacy fixture also omitted designated timestamps; adding +them made the frame valid without changing the behavior under test. + +Red evidence uses the frozen phase-5a client JAR. The four original +LONG-to-decimal methods all reject the missing conversion; the new public +fixture's two acceptance tests fail while its independent raw legacy +characterization passes. Final client focused validation passes three tests +plus two packaging checks; canonical validation passes 475 tests plus two +packaging checks. Install passes, and workspace, archived and installed client +JARs all have SHA-256 +`41d137a1c2388ed21f776e372771785377d91c15a887d94f04e82c97f505d403`. + +Final server focused validation passes seven tests and the canonical gate +passes 345. The unchanged original 333-test diagnostic reports zero failures +and 14 errors: exactly the four intended LONG-to-decimal methods disappeared +from phase 5a's 18, with no additions. Three seeded repeats pass 108 tests each +for seeds 1/2, 42/43 and 1234567/7654321; their selected XML reports preserve +the exact seed, project-local temporary directory, installed client artifact +and zero failures/errors/skips. No server production or SQL-engine file changed +for this increment. + +The performance check targets the changed common LONG binding path, not decimal +ingestion. A pre-edit fixed-work study and 12 balanced control/candidate runs +each process 500 million decoded known-schema LONG-to-LONG writes with identical +checksum `2715042977495835685`. Paired instructions change by +0.003% to +0.004%; +paired cycles change by -0.164% to +2.237%, below the declared repeatable 5% +guard. All measured runs have zero major faults and 100% counter coverage, with +no run-window cgroup throttle/OOM delta. The hardware-study validator passes. +This supports only that no material regression was detected in LONG-to-LONG; +it makes no decimal, network, ingestion or general-throughput claim. + +Final acceptance leaves 14 original diagnostic errors: STRING-to-decimal, +FLOAT/DOUBLE-to-decimal and native-decimal-to-text. No commit or push is part of +this checkpoint. + +## D050 — STRING-to-decimal conversion + +Status: iteration 2.34 and phase 5c accepted, 2026-09-11. Revision 52 records +the bounded scope: public `stringColumn` now reaches every +DECIMAL8/16/32/64/128/256 target in schema mode. Phases 5d–5e remain; this is +neither complete conversion coverage nor release readiness. + +The server parser and all six decimal target paths were read before integration. +For a known decimal target, the client parses with that target's precision and +scale, `strict=false` and `lossy=false`, preserving the server grammar's exponent +underscores, then emits the QWP decimal wire type for the target storage class; +DECIMAL8/16/32/64 share QWP's DECIMAL64 wire type. The shared client +parser now uses the current server's leading-zero and full-scale precision +calculation. This broad precision-metadata correction required updating 19 stale +assertions in 16 existing tests; the parsed values and scales are unchanged. + +The stable schema-mode null rule is independent of target width: Java null and +exact optionally signed `NaN` or `Infinity`, after supported suffix processing, +emit target-scale bitmap SQL null. Malformed special-value prefixes and extreme +nonzero integer-boundary exponents fail locally as `INVALID_VALUE`. This +avoids the server's small-decimal special-null narrowing bug. The exact-special, +precision and exponent corrections are global shared-parser changes, not only +schema-mode behavior: direct `Decimal64`, `Decimal128` and `Decimal256` string +parsing now rejects malformed special prefixes and reports extreme nonzero +exponents as `NumericException`. Confirmed MISSING and legacy Sender paths remain +VARCHAR and use server-side conversion. + +The implementation reuses the existing per-column Decimal256 scratch and the +phase-5a precision/storage/direct-width append tail. Exact special checking is +corrected centrally according to the parser's documented contract. No separate +schema parser, conversion registry, retained state, protocol, server production, +SQL or send-time transformation was added. + +The shared client/server corpus has 55 rows: 19 values, 24 invalid inputs and +12 nulls. Both copies have SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`. +Client red validation first failed eight tests. Final focused validation passes +eight tests plus two packaging checks, all-decimal validation passes 463 tests, +and the canonical gate passes 941. Candidate, archived and installed JARs have +SHA-256 +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +A 3,601,944-case differential passes ordinary server parity and records 72 +deliberate malformed-prefix corrections. + +Server focused validation passes seven tests and the canonical gate passes 352. +The unchanged original 56-test conversion class reports ten errors: exactly the +four STRING-to-decimal methods disappeared from phase 5b's 14, with no additions. +Three seeded repeats pass 115 tests and 22 selected XML reports each. + +The performance check targets the changed native STRING binding path, not +decimal ingestion. Twelve balanced fixed-work native STRING-to-VARCHAR runs have +identical checksum `15640344951909530213`. Paired basic instructions change by +-0.001% to -0.000% and paired cycles by -1.028% to -0.007%, with no repeatable +5% regression. This makes no decimal, network, ingestion or general-throughput +claim. + +Final acceptance leaves ten original errors: FLOAT/DOUBLE-to-decimal and +native-decimal-to-text. No commit or push is part of this checkpoint. + +## D051 — FLOAT/DOUBLE-to-decimal conversion + +Status: iteration 2.35 and phase 5d accepted, 2026-09-12. Revision 53 records +the bounded scope: public `floatColumn` and `doubleColumn` now reach every +DECIMAL8/16/32/64/128/256 target in schema mode. Phase 5e remains; this is +neither complete conversion coverage nor release readiness. + +Root and two SOL reviewers traced the real server route before integration. +`WalColumnarRowAppender` reads FLOAT as a Java float and widens it to double; +both source widths then call `Numbers.doubleToDecimal(value, precision, scale, +false)`. That path formats the shortest round-tripping decimal and rejects a +natural scale above the target instead of rounding or truncating. Target +precision overflow also rejects. All six server target loops map NaN and both +infinities to decimal null. SQL casts use a different lossy setting and were not +used as the oracle. + +The client follows that contract directly. FLOAT widens through the existing +double-valued binding path. Finite values reuse the schema-only Ryu formatter, +the shared decimal parser at the target precision/scale, the existing per-column +Decimal256 scratch and phase 5a's direct target-width append tail. Small decimal +targets use DECIMAL64 wire storage while retaining their declared precision and +scale. Non-finite values use the target-scale decimal bitmap-null helper. +Duplicate suppression still precedes conversion, and failures remain +nonretryable `INVALID_VALUE` errors that permit whole-row rollback. Confirmed +MISSING and legacy mode keep native FLOAT/DOUBLE tags and server-side conversion. +No parser, state, conversion registry, protocol, server production, SQL or +send-time transformation was added. + +The first correct implementation placed the decimal branch in the common +floating numeric helper. Functional tests passed, but the hardware guard found +a real native-path cost: the helper grew from 311 to 411 bytecodes, exceeded +HotSpot's hot inlining threshold and increased instructions by about 4.7% for +DOUBLE-to-DOUBLE and 3.8% for FLOAT-to-FLOAT. The implementation was simplified +before acceptance: numeric targets retain their old branch and bytecode shape, +while decimal dispatch lives at the top of the existing cold nonnumeric helper. +No production guard or abstraction was added. + +The final fixed-work A/B study contains six counterbalanced control/candidate +pairs for each native path, 500 million writes per run. DOUBLE checksum is +`11101713426464862245`; median paired elapsed, cycles and instructions changes +are -1.36%, -1.04% and +0.0037%. FLOAT checksum is `3646854813611232293`; +medians are -0.95%, -0.83% and +0.0001%. All 24 runs have identical work and +checksums, zero major faults, complete counters and no cgroup throttle/OOM delta. +Both hardware-study validators pass. This supports only that no material native +binding regression was detected; it is not decimal, allocation, network, +ingestion or general-throughput evidence. + +The server-generated client/server corpus has 60 rows: 24 FLOAT and 36 DOUBLE; +37 values, 17 invalid values and six nulls. It covers all twelve source/target- +class pairs, exact wire width, scale and limbs, no-rounding boundaries, widened +float `0.1`, signed zero, non-finite values, `Float.MIN_VALUE` scale 60/59, +underflow and extreme overflow. Both copies have SHA-256 +`366a889719c6e7b688ff00e1bdfb95e273ab192508a1094dac5312c3d899611d`. +The generator calls the current server `Numbers.doubleToDecimal(..., false)`; +its output is byte-identical to both resources. An independent differential +checks 5,996,868 finite target/value combinations and matches acceptance, scale +and all four limbs. + +Component tests consume every vector and assert schema flag, physical target +type, bitmap, target scale, limbs and frame length. A second test covers +duplicate-first-wins, non-finite-null-first, omission, reset and A/failed-B/C +rollback. The public socket test changes DECIMAL64(18,4) to DECIMAL128(20,5): +the old block stays pinned while a value invalid under the old snapshot causes +one refresh and succeeds under the new snapshot. Final focused client validation +passes 63 tests plus both packaging checks. The corrected canonical gate passes +944 tests across 46 classes. Candidate, archive and installed JAR SHA-256 is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +The final binding source SHA-256 is +`2f8e23ba68fb415a8ff9ca7b5535c9f27d040a7b9d39541212baec0310efa1ea`. + +Four real-server E2Es consume the same corpus through public Sender, assert exact +SQL/null results and local typed errors/recovery, preserve native inference for +a missing table, and independently send an unflagged raw legacy frame with and +without null bitmaps. They inspect native tags, raw bits and bitmap context before +the server performs its old conversion. A separate server rebind or store-and- +forward fixture was not added: the new client socket test already pins the +width/scale generation, while existing decimal and schema replay tests exercise +persisted direct decimal bytes. Adding another scenario would repeat machinery +without exposing a phase-specific behavior. + +Server focused validation passes the four new E2Es and four recovered original +methods. The canonical gate passes 360 tests. The unchanged original 56-test +conversion diagnostic has zero failures and six errors: exactly the four phase- +5d methods disappeared from phase 5c's ten, with no additions. The remaining six +are the phase-5e native-decimal-to-STRING/VARCHAR methods. Three seeded repeats +pass 123 tests each for seeds 1/2, 42/43 and 1234567/7654321; each archived +selection has 23 XML reports with the requested seeds, project-local temporary +directory and zero failures/errors/skips. The installed JAR hash remained the +accepted candidate throughout server validation. No server production or SQL +engine file changed for this increment. + +Two failed validation setups are retained but excluded. The first 60-table +server run used the nearly full shared `/tmp`; WAL mmap creation hit errno 28 on +the last two tables. The complete project-local-temp rerun passed all four tests. +The first client canonical reconstruction incorrectly read every stale XML file +in the phase-5c evidence directory instead of the 45 classes actually named in +its log. That exploratory run executed 1,278 tests and exposed four errors in +`CursorWebSocketSendLoopErrorLatchTest`: the test used +`Unsafe.allocateInstance`, skipped the real final schema-coordinator initializer +and then invoked private `recordFatal` by reflection. Adding a production null +guard would have served only this impossible object state. The obsolete 251-line +test was deleted after SOL review; seven public/real-loop replacement classes +pass 12 tests, and the expanded 79-class regression gate passes 1,269 tests. +The exact prior-log-plus-new canonical selector then passes 944. This cleanup is +not evidence for the conversion itself, but it preserves the design rule against +reflection fixtures and avoids defensive production code. + +The review identified one sibling fixture with the same bare-loop pattern. +`CursorWebSocketSendLoopJvmErrorTest` left the engine null, so its private +`ioLoop` case entered a zero-backoff retry and the isolated run timed out after +60 seconds with millions of attempts. The legitimate public +`connectWithRetry` test remains; only the two reflective private-loop cases and +their fake-object plumbing were removed. The retained test passes. A normally +constructed real-loop test now covers one-attempt JVM-error escape, producer- +visible terminal wrapping and clean I/O-thread shutdown without reflection or a +test hook. No production change was made for this impossible object state. + +Client HEAD remains `981bdb02a471f3b290c89b8e78cbc422610e329e`; server HEAD +remains `12a33d651e51e2682e7a448c8db5168fc72dfad3`, both with the recorded +dirty worktrees. Final acceptance leaves six original diagnostic errors, all in +native-decimal-to-text phase 5e. No commit or push is part of this checkpoint. + +## D052 — Native-decimal-to-text contract and malformed-wire boundary + +Status: phase 5e prepared, not yet accepted, 2026-09-12. Revision 54 records the +contract before production code changes. The remaining six original failures +are `Decimal64`, `Decimal128` and `Decimal256` into STRING and VARCHAR. + +The server target dispatch promotes each native wire width to its reusable +Decimal256 value, applies the unsigned source wire scale and calls `toSink`. +The output is plain fixed-point text with valid source scales and trailing zeros +preserved. Schema mode will therefore promote a non-null public decimal into a +reused binding-owned Decimal256, apply the same unsigned one-byte scale, format +it into the reused text sink and append the existing `TYPE_VARCHAR` +representation for both text targets. MISSING and legacy mode remain native +decimal. Public null and `NULL_VALUE` inputs retain their existing no-op +semantics and do not claim a duplicate slot. No protocol, server, SQL, +buffer-state, conversion-registry or send-time change is justified. + +The compatibility boundary is explicit. A raw non-bitmap DECIMAL64/128 sentinel +is currently formatted by the server as the physical minimum integer; the raw +DECIMAL256 sentinel tuple produces an empty non-null string. Public decimal null +sentinels remain setter no-ops. Public low-level constructors and `setScale` +mutators can, however, construct other coefficients and scales outside the +documented 18/38/76-digit domains. The first implementation incorrectly called +each narrow concrete formatter directly; a full-width Decimal128 value was +silently truncated because the server formats it only after Decimal256 +promotion. That implementation is rejected. The corrected path mirrors the +server's promotion and unsigned one-byte scale instead of adding a new local +rejection policy. Hardening the Decimal API or server validation would be a +separate production decision. + +The frozen phase-5d client artifact is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +Against it, the unchanged real-server diagnostic fails exactly the six phase-5e +methods with unsupported conversion errors. One earlier setup failed before +tests because its exact project temporary directory did not exist; that log is +retained as invalid setup evidence and is excluded. Shared corpus, component, +public Sender and independent legacy-frame tests must be red against this frozen +artifact before the narrow binding branch is implemented. + +## D053 — Promote native decimals before text formatting + +Status: accepted in iteration 2.36, 2026-09-12. +Revision 55 supersedes D052's initial direct-formatter assumption. + +Adversarial review found that public two-argument raw constructors and low-level +`setScale` mutators can expose values outside the narrow classes' documented +precision and scale domains. Calling `Decimal128.toSink` directly silently +changed the full physical value `2^127 - 1` into 38 nines, while the server first +promotes it and emits all 39 digits. Adding a new local rejection policy would +also change previously accepted old-server writes and is unnecessary. + +The corrected binding lazily reuses one Decimal256 scratch, copies every native +decimal into it, applies the unsigned low QWP scale byte, then formats into the +existing reused text sink. This is the server's scalar algorithm. It reuses its +scratch and sink; after the sink reaches the required high-water capacity, +formatting needs no per-value object allocation. Public null sentinels remain +no-ops. + +Parity is deliberately per setter value, not per legacy column block. A known +STRING/VARCHAR target accepts mixed native widths and independently preserves +each value's effective scale. Legacy inference must choose one native wire width +and scale for the whole block, so it rejects a later width and may rescale or +reject a later scale. Recreating that intermediate constraint after the target +is known to be text would add buffer state and make a value's conversion depend +on neighboring rows; revision 55 rejects that complexity. Tests must name and +cover this behavior difference directly rather than calling it whole-batch +legacy parity. + +D052's proposed separate component fixture is also superseded. The public +socket test already drives the binding and asserts the exact bitmap, offsets, +bytes and end-of-frame boundary. Repeating that path through a lower-level +fixture would add test machinery without another observable contract. + +The first candidate JAR with direct narrow formatting and SHA-256 +`4af31d76224a3e811527114638240eabfe5939445f218c798c7f2fc3d1247b9e` +is rejected and retained only as superseded evidence. The accepted candidate +and installed JAR is +`e06759605ee3d1cf01c3277ba13531b0172e42138042c1f2f49c2db49e04b22d`; +only `QwpSchemaBinding.class` differs from the frozen phase-5d JAR. + +The stored production/server differential passes 3,000,294 deterministic +comparisons across full physical limbs, arbitrary integer scales, explicit +scale boundaries and all three public null sentinels. The 24-row shared corpus +is byte-identical in both repositories. Client validation passes the 46-test +public integration class, including four phase-5e tests, the 950-test canonical +gate, 1,275-test broad gate and two packaged-JAR tests. All 80 selected broad +XML reports have zero failures, errors and skips. + +Real-server validation passes all three Phase-5e E2Es, including mixed widths/ +scales, the full physical Decimal128 value and wrapped scale 257. The unchanged +original conversion diagnostic passes all 56 methods, resolving the original +47 failures. The accumulated server gate passes 363 tests. Three seeded repeats +pass 126 tests each; all 24 selected reports per run contain the requested seeds +and project-local temporary path with zero failures, errors and skips. No server +production or SQL-engine change belongs to phase 5e. + +The clean performance window contains six counterbalanced pairs for each native +identity width and 200 million setter calls per process. Instructions change by ++1.05% for Decimal64, +1.38% for Decimal128 and +0.71% for Decimal256; paired +median cycles change by -1.93%, +1.13% and -1.03%. Exact work and checksums +match, counters run at 100%, and no run has GC, major faults, throttling or OOM +changes. The result accepts the change against the 5% regression threshold but +makes no speed, decimal-to-text, network or ingestion-throughput claim. One +Maven-overlapped matrix and the withdrawn-candidate matrix are isolated and +excluded. Exact commands, hashes and logs are in the client and server 2.36 run +manifests. No commit or push is part of this checkpoint. + +Final independent SOL acceptance found no P0, P1 or P2 blocker after checking +production ownership, compatibility wording, exact hashes, raw logs, XML +provenance and performance claim bounds. One P3 remains: older source-map line +numbers in the design have drifted, although their method names remain correct. +That navigation debt is outside phase 5e and does not change the contract. + +## D054 — Post-2.36 simplification and next-iteration plan + +Status: accepted planning decision, 2026-09-14. Revision 56 records the plan; +it does not claim release acceptance or add production code. + +The schema-aware implementation is now committed as client `6fe3ce44` and +server `ffbb7125d2`. Server integration commit `818608c297` pins that exact +client revision. A Maven reactor run using the pinned local client passed the +public-Sender UUID conversion and rollback E2E against the server, so the next +work starts from an integrated baseline rather than another component-only +slice. + +Compatibility becomes a permanent CI and release gate, not another feature +iteration. It must exercise released binaries for new-client/old-server and +old-client/new-server behavior, current new/new schema mode, and the one-way +upgrade failure against an old endpoint. This validates the promised behavior +without adding a compatibility adapter, downgrade encoder or send-time +transformation. + +Timeout productization is removed from the roadmap. The policy is a fixed +30-second deadline with no new public configuration: initial negotiation and +its first describe share one budget, while later cache-miss describes and +refreshes each get their own budget. The timeout change is owned separately +from these conversion slices and is implemented by client commit `98be3b0b`; +it is not recreated or extended by iteration 2.37. + +The next implementation areas are deliberately narrow: iteration 2.37 adds +BYTE/SHORT/INT to the six numeric targets with a small shared helper; iteration +group 2.38 adds fixed-width identity/text support as separate IPv4, LONG256 and +exact-precision GEOHASH slices. Each slice retains a real public-Sender E2E and +the existing wire, SQL, rollback, refresh, recovery, packaging and +affected-performance gates. + +The source audit remains broader than the intended product. BINARY-to-parser +paths, LONG_ARRAY, non-ASCII CHAR-to-VARCHAR and malformed decimal metadata are +server-contract questions, not automatic client backlog. The client will not +reproduce parser accidents or validation gaps merely because current server +dispatch can reach them. The encoded 32-bit `ColumnType` remains the single +parameter representation, and no converter registry, schema patch protocol, +retry framework, second request channel or per-path timeout setting is planned. + +Before iteration 2.37 code lands, tests must freeze `Integer.MIN_VALUE` behavior +with and without a companion null bitmap. The recommended schema-mode contract +is stable source NULL, independent of neighboring rows; this remains an explicit +contract checkpoint rather than an assumption copied from LONG. + +## D055 — Small integer inputs use direct numeric target widths + +Status: iteration 2.37 implemented and locally validated, 2026-09-14. Revision +57 is recorded in the client commit containing this decision; the companion +server commit pins that exact client revision. + +BYTE, SHORT and INT setters now support BYTE, SHORT, INT, LONG, FLOAT and DOUBLE +targets. One private helper owns their shared numeric-target check, narrowing +range checks and direct append. The public Sender branches reuse the established +schema-rejection rollback and one-refresh path. Missing columns retain the +setter's native source type. Legacy mode and the established LONG implementation +are unchanged; no registry, converter graph, new buffer state or server +production change was added. + +The source-null decision is now explicit. `Integer.MIN_VALUE` is a stable INT +source null in schema mode, independent of whether another row creates a null +bitmap. BYTE and SHORT have no public source-null sentinel. An unsupported +target still rejects the setter even when the INT value is the null sentinel; +null does not make an otherwise unsupported conversion available. + +Both repositories consume the same 64-case boundary corpus, SHA-256 +`93dd3987145766e6b9d7fff0b592e0a56b67577d83de4de623ed0e8231e3b96d`. +It covers identity, widening, valid and invalid narrowing, FLOAT/DOUBLE raw-bit +rounding and INT nulls. Client component tests assert exact target wire data and +bitmap behavior. Public scripted-Sender tests add partial-row rollback, one +refresh, snapshot pinning and missing-column inference. A public Sender restart +test retains an unacknowledged schema frame and proves byte-identical replay. +The real-server E2E creates all six numeric target tables and verifies the +accepted values through SQL while range failures remain local typed errors. + +Focused client validation passes 91 tests: 35 binding, 47 public Sender and 9 +network replay tests. The established client gate passes 957 tests, followed by +both packaged-JAR integration checks. The focused real-server E2E passes, and +the corrected accumulated server selector passes 379 tests with zero failures, +errors or skips. + +One exploratory server run incorrectly selected every method in +`QwpSenderE2ETest` and `QwpWebSocketSenderReceiverTest`. It ran 631 tests and +reported 3 failures plus 65 errors in conversion families that remain explicitly +unsupported, chiefly arrays, geohashes and parser paths. That run is not a +regression result. A second invalid setup exhausted shared `/tmp`; its generated +512 MiB test directory and crash log were moved under the run-owned server +target directory without touching unrelated temporary files. The valid run uses +project-local temporary storage and only the intentionally supported legacy +methods from the established acceptance selector. + +The affected fixed-work guard runs each identity source for two million warmup +and 50 million measured rows, five times on one CPU. All checksums and work +counts match and hardware counters run at 100%. Average retired instructions are +30.45 billion for BYTE, 30.76 billion for SHORT, 32.54 billion for INT and 35.14 +billion for the established LONG reference. A separate 100-million-row INT run +records no GC after startup. These data reject an obvious helper-path regression; +they are not a before/after speed claim or network/ingestion benchmark because +the prior client had no corresponding schema-aware small-integer operation. + +The fixed 30-second timeout remains outside the conversion roadmap. Iteration +2.37 is rebased on the user's timeout commit `98be3b0b` and does not duplicate +or extend that change. + +## D056 — Released binaries form the standing compatibility gate + +Status: implemented and locally validated, 2026-09-14. Design revision 58. + +Compatibility is now a build gate rather than another conversion iteration. +The gate adds test and CI files only; it does not change client or server +production code, add a protocol adapter, or transform buffered data at send +time. It runs these three process combinations: + +1. current client against QuestDB 10.0.1; +2. client 1.3.9 against the current server; and +3. current client against the current server. + +QuestDB 10.0.1 and client 1.3.9 are the fixed last released baseline before the +schema extension. Their tagged sources contain no schema negotiation headers, +and QuestDB 10.0.1 itself pins client 1.3.9. The peeled source tags are server +`7a391566ac827e0d8c269b91f2415c16b0a32a84` and client +`3d035bdcab9d65b487aa16ffcee2fbd608e84d74`. The gate downloads the +published JARs into its own target directory and verifies SHA-256 +`432d27433836d4430aaca4d701cb0dfac23f7b8281963a5fae97079c0436892d` +for the server and +`41c31bc25646cfe1f045615f53edcfa685a01182749d96c381ea75cd27e7a6a0` +for the client. The shared SLF4J API 2.0.17 dependency is pinned to +`7b751d952061954d5abfed7181c1f645d336091b679891591d63329c622eb832`. +These are deliberate baseline pins, not moving `latest` lookups. + +One Java probe is compiled with `--release 11` against client 1.3.9's public +`Sender` API, then launched in isolated JVMs with exactly one selected client +JAR. It writes one buffered three-row block to a table whose `value` column is +FLOAT: `42`, `Long.MIN_VALUE`, and an omitted value. Explicit auto-flush limits +keep the rows in one block because the last omission creates the bitmap that +makes the old LONG sentinel behavior observable. Legacy mode stores `42.0`, +`-9.223372E18`, and SQL NULL. Schema mode stores `42.0`, SQL NULL, and SQL NULL. +Exact exported CSV distinguishes the modes; mere write success would not. + +The runner starts each server with separate roots and ports, waits for HTTP +readiness, creates WAL tables through the public HTTP endpoint, executes the +probe, polls exported rows for WAL visibility, and retains per-run server logs. +Released artifacts live under `compat/target`, outside the shared Maven cache. +The default source is the canonical Maven Central endpoint; an internal mirror +can be supplied without bypassing checksum validation. + +The same script runs after the existing distribution build in +`.github/workflows/pgwire_stable.yml` and in the Linux release build before +artifacts are copied. It reuses the current server JAR and, when the server pins +a client snapshot, the exact checked-out submodule JAR already built by the job. +When the server pins a released client, it uses that exact resolved dependency +from the local Maven cache. A cold local run, including roughly 40 MiB of +released artifact downloads, passed all three combinations in 5.03 seconds. A +focused client run also passed 60 negotiation, Sender-integration and replay +tests with zero failures, errors or skips. The corresponding focused server run +passed 15 schema discovery, numeric-conversion and identity tests with zero +failures, errors or skips using project-local temporary storage. An initial run +against the already-full shared `/tmp` failed with `No space left`; it is +environmental evidence, not a product regression. + +The process matrix intentionally does not recreate the upgraded-sender state +transition with two server processes. Deterministic socket and replay tests +already prove the stronger exact contract: after schema confirmation, an old +endpoint receives no schema frame, durable and ordinary watermarks do not +advance, and retained bytes replay unchanged to a compatible endpoint. Keeping +that assertion there avoids a timing-sensitive failover rig while the process +gate proves released-binary interoperability and observable mode selection. + +Three independent SOL reviews checked the contract, published artifacts and +CI harness. The first review found an unbounded server `wait`, a probe without +a kill grace period and wording that overstated the public-API level of the +existing watermark tests. The CI review found that Azure's custom Linux +condition replaced its default success gate, that Azure did not retain failure +logs, and that fixed ports plus iteration-counted polling left avoidable false +target and timeout risks. Root replaced those with bounded TERM-to-KILL +shutdown, a probe kill grace period, five-minute CI limits, a success-preserving +Azure condition and failure artifact, occupied-port rejection, real 30-second +deadlines, and accurate socket/replay wording. Released client pins now select +the exact Maven-cache artifact while snapshots select the checked-out submodule +build. A negative run substituting the old client for the current client failed +the schema assertion and cleaned up both servers. Final re-review reports no +blocker or remaining concrete finding. + +## D057 — Textual decimal input reuses Decimal256 conversion + +Status: iteration 2.38 implemented and locally validated, 2026-09-14. No commit +or parent-repository submodule update has been made. + +The public `decimalColumn(CharSequence)` overload now participates in schema +mode. It remains a DECIMAL256 input, not an alias for `stringColumn`: the Sender +parses once into its existing Decimal256 scratch value, and the binding reuses +the native decimal target conversion and text-formatting tails. Known decimal +targets apply their declared precision and scale. STRING and VARCHAR receive +canonical fixed-point text. This adds no converter registry, retained per-row +object or server production change. + +Target lookup, duplicate suppression and target support checks precede parsing. +An unsupported known target therefore reports `UNSUPPORTED_FEATURE` even for +malformed input, while a duplicate malformed value remains ignored. Java null +and empty input remain no-ops. Parsed `NaN` and infinities remain effective +values that write target NULL; this differs intentionally from typed decimal +null sentinels, which remain no-ops. The public Sender uses the established +partial-row rollback and one-refresh path for local schema errors. + +A confirmed missing column infers DECIMAL256 at the first finite input's natural +scale. A preceding parsed special must not pin scale zero. If a block contains +only decimal nulls, `QwpTableBuffer` emits legal scale-zero wire metadata while +retaining its internal unset scale, so a later finite batch can still choose its +natural scale. This is an encoding-boundary normalization, not new scale state. + +Tests use public behavior and real wire/data results, without reflection or +test-only production hooks. On JDK 25, 114 affected client tests and both +packaged-JAR checks pass with the fork's startup temp root under +`/mnt/pcie5/schema-aware-decimal-text-20260914/client/jdk25-verify`. They cover +all six decimal widths, STRING/VARCHAR formatting, missing inference, parsed +specials, null/empty omission, duplicate ordering, invalid-row rollback, +refresh and exact frames. The companion server selection passes 12 tests for +the new overload plus existing native/string decimal paths under +`/mnt/pcie5/schema-aware-decimal-text-20260914/server/regression`. After +replacing deterministic `returnsOnce()` assertions with the full query battery, +the three new real-server tests pass again under the `server/final3` directory. + +Early server attempts inherited a full shared `/tmp`; server logs proved WAL +mmap failures with `No space left`. No product change followed from those runs. +After the test scratch location was moved to `/mnt/pcie5`, all subsequent test +databases and scratch data used that filesystem. It remains required for future +validation. + +Two broader checks exposed existing branch constraints rather than decimal +failures. First, the Java 8 build stops in the committed, untouched +`QwpSchemaRyuDouble` because it calls Java 9+ `Math.multiplyHigh`; the new code +uses Java 8 syntax and APIs, but a real Java 8 build remains a release gate after +that baseline blocker is repaired. Second, the full client suite blocks in +offline-first tests because the current mandatory initial schema lookup waits +before accepting the first value. The exact recovery failure reproduces at +client baseline `1310b0dd`, and `CloseDrainTest` reaches the same wait on the +current tree. This is the unresolved product conflict between cold offline +buffering, mandatory local schema conversion and the prohibition on send-time +transformation. The simplest coherent contract is to require the first +effective write to establish schema mode; changing that contract needs a +separate decision and is not hidden by excluding individual tests. + +No setter-specific SF replay test was added. This slice changes how the Sender +constructs an ordinary schema-pinned table buffer, not persistence or replay; +the public exact-frame tests cover the newly constructed bytes and the existing +schema-frame replay tests remain byte-oriented. Adding another replay scenario +would duplicate those two contracts without exercising new recovery logic. + +An author and an independent SOL reviewer challenged the first draft. They +found and fixed missing-column dispatch through an unparameterized DECIMAL256 +type and a special-null path that prematurely locked scale zero. Final review +found no implementation blocker, confirmed no apparent per-row heap allocation, +and corrected the affected-test count from an earlier unsubstantiated 218 to +the preserved 114-test reports. + +## D058 — IPv4 uses target-native wire data + +Status: iteration 2.39a implemented and locally validated, 2026-09-14. Design +revision 60. The parent-repository test and submodule update land separately. + +Both public `ipv4Column` overloads now participate in schema mode. IPv4 targets +retain the four-byte packed value. STRING and VARCHAR targets receive canonical +dotted-quad text through the binding's existing reusable numeric sink. The +implementation adds no converter registry, address object, parser fork or +per-row string allocation. Legacy mode keeps its existing native IPv4 wire path. + +Packed zero is the public IPv4 null sentinel and becomes an explicit bitmap +NULL for every target. A Java null text reference remains a no-op. The +case-insensitive `"null"` literal and exact `"0.0.0.0"` remain rejected, while +legacy dotted aliases such as `".0.0.0.0."` remain accepted and normalize to +NULL. Target selection and duplicate suppression precede parsing, so an invalid +duplicate stays ignored and an unsupported known target wins over malformed +text. Sender-level schema errors use the established partial-row rollback and +single-refresh lifecycle. + +The implementation review found one pre-existing parser boundary: an address +made only of dots read past the input and leaked `StringIndexOutOfBoundsException` +instead of the public invalid-value error. One bounds check now converts that +case to `NumericException`; `"."` and `"...."` are fixed shared-corpus cases. +The permissive leading/trailing-dot grammar is otherwise unchanged. + +The 16-row corpus covers packed boundaries, canonical and normalized text, +zero/null behavior and malformed input against IPv4, STRING and VARCHAR +targets. Component tests assert exact wire bytes and bitmaps, inference, +omission, duplicate ordering, unsupported/parameterized/designated targets and +rollback. Public scripted-socket tests assert the same target wire plus a schema +change from IPv4 to VARCHAR while the already-buffered block remains native +IPv4. A separate old-peer test proves the integer and text overloads still emit +one unflagged native IPv4 frame without a DESCRIBE. The real-server test sends +all three targets in one batch, verifies exact stored values, local invalid-row +recovery and missing-table inference. A compact store-and-forward test inspects +the persisted frame before replay and verifies the same rows after replay. + +On JDK 25, 238 focused client tests pass with zero failures, errors or skips: +the schema binding, Sender integration, legacy Sender, QWP constants and numeric +parser suites. +Five focused server tests pass against the installed client snapshot: the three +new IPv4 tests plus existing auto-create and schema-aware text-target tests. All +validation used `/mnt/pcie5/schema-aware-ipv4-20260914`. An earlier run inherited +the full shared `/tmp`; logs showed all three WAL writers accepted eight rows, +then STRING and VARCHAR WAL apply suspended on `errno=28`. That run is discarded +as environmental evidence. Setting the forked JVM's `java.io.tmpdir` moved the +JUnit database roots to the project scratch filesystem and the same single-flush +test passed. + +The final SOL review found no production correctness, compatibility, allocation +or complexity blocker. It corrected one design sentence that named the server's +IPv4 formatter as though it existed in the client, and requested the explicit +old-peer setter test above. Re-review found both corrections complete. + +## D059 — LONG256 uses target-native wire data + +Status: iteration 2.39b implemented and locally validated, 2026-09-14. Design +revision 61. The pre-iteration baseline is client `21168def` and server +`d7b94647ae`; neither repository is committed by this decision record. + +The public `long256Column` setter now participates in schema mode. A LONG256 +target receives the four input limbs directly in the existing 32-byte wire +layout. STRING and VARCHAR targets receive direct VARCHAR wire data in the +server's canonical lowercase `0x` form: the most-significant nonzero limb is +written first with even-byte leading width, and each lower limb is padded to 16 +hex digits. The implementation keeps four primitive longs and reuses the +binding's numeric text sink and existing hex appenders. It adds no `BigInteger`, +LONG256 object, converter registry, retained source value, server production +change or per-row string allocation. + +Four `Long.MIN_VALUE` limbs are the source null sentinel in schema mode and are +encoded as an explicit bitmap NULL for LONG256, STRING and VARCHAR targets. Any +partial match remains a value. This removes a legacy row-context dependency: +without a bitmap the server cursor recognizes the four-limb sentinel as null, +but with a bitmap it treats only marked rows as null, so the same unmarked +sentinel converted to text becomes an empty non-null string. Schema mode makes +the intended null explicit before target conversion. Legacy mode deliberately +keeps its native LONG256 wire data and existing server-side behavior. The public +Javadoc states this qualification instead of promising uniform legacy results. + +Target selection and duplicate suppression happen before formatting. Known +unsupported, parameterized and designated targets keep typed rejection; +Sender applies the established partial-row rollback and one-refresh lifecycle. +A confirmed missing column still infers native LONG256. A later schema +generation changes only subsequent buffers: exact scripted-wire coverage proves +an already buffered LONG256 block stays native before a new VARCHAR block is +created. An old peer receives the original unflagged native LONG256 frame and no +DESCRIBE. + +The shared eight-row corpus covers zero, the highest nonzero `l0`, `l1`, `l2` +and `l3` formatter branches, all-one limbs, a partial sentinel, the high sign +bit and the full sentinel null. The client and server resources are +byte-identical with SHA-256 +`a634074af646eb961768599e73727a4a1ab0cd0916dc97a86a8ee4a027723275`. +Component tests assert exact target type, payload and bitmap bytes plus +duplicate, omission, rollback, inference and rejection order. Public socket +tests cover exact schema and legacy frames, rollback and generation rebind. The +real-server corpus uses one Sender and one flush across LONG256, STRING and +VARCHAR tables, with an omitted first value to force late column creation and +two explicit bitmap-null positions. Separate E2E tests cover A/error/C row +recovery, missing-table inference and inspection plus replay of the persisted +target-native frame. A raw legacy fixture pins the bitmap-dependent server +behavior instead of making the schema test silently replace it. + +Final focused validation on JDK 25 passes 96 client tests and nine server tests +with zero failures, errors or skips. The client set is +`QwpSchemaBindingTest,QwpSchemaSenderIntegrationTest`. The server set is +`QwpSchemaLong256E2ETest,QwpSchemaTextSourceNullE2ETest` plus +`QwpSenderE2ETest.testSchemaModePreservesUuidAndLong256NullsForTextTargets`. +An expanded client regression covering the table buffer, encoder, Sender, +constants, prior STRING-to-LONG256 and UUID-to-text bindings, recovered-frame +analysis and schema replay passes 283 tests with zero failures, errors or skips. +All test roots were created under `/mnt/pcie5` by setting `java.io.tmpdir` at +test-JVM startup through `JAVA_TOOL_OPTIONS`. + +An exploratory run of the full `QwpSenderE2ETest` is not green: 137 tests +produced three failures and 41 errors. The LONG256 test changed by this slice +passed. Observed errors include still-missing inventory entries such as +INT-to-IPv4, STRING-to-DATE and LONG_ARRAY, but this run was not fully +classified and is not presented as regression evidence. It confirms the +standing release status: the conversion contract remains incomplete even +though the bounded LONG256 slice is accepted. + +The first server corpus run appeared to return an empty table. The server log +disproved a QWP rejection: LONG256 had applied all eight rows, while the next +STRING WAL apply failed with `errno=28` because shared `/tmp` was full. Passing +`-Djava.io.tmpdir` as an ordinary Maven property did not fix the environment; +`strace -f -yy -e trace=fallocate` showed that Surefire's JVM had cached +`/tmp/junit...` before that property was applied. Supplying it at JVM startup +moved the actual database files to `/mnt/pcie5`, where all three eight-row WAL +applications and queries passed before the final corpus branch was added. The +final nine-row version passes as part of the focused server gate. No production +or test-shape workaround follows from the environmental failure. In particular, +the final test restores the simpler and stronger one-Sender, one-flush shape. + +Independent adversarial review found and closed two issues before acceptance: +the initial Javadoc overstated legacy null semantics, and the seven-row corpus +missed the separate `l1` formatting branch. Re-review found both resolved and no +remaining correctness, compatibility, allocation or complexity issue in this +slice. The wider conversion inventory remains open; the next bounded slice is +native GEOHASH identity and text output with exact target precision. + +## D060 — GEOHASH uses exact native precision or fixed-width text + +Status: iteration 2.39c implemented and locally validated, 2026-09-14. Design +revision 62. The pre-iteration baseline is client `0d9663b26f` and server +`de96ef5eb5`; neither repository is committed by this decision record. + +Both public `geoHashColumn` overloads now participate in schema mode. A GEOHASH +target receives the masked packed value only when the target and source bit +precisions match exactly; a mismatch fails locally with `UNSUPPORTED_FEATURE`. +STRING and VARCHAR targets receive direct VARCHAR wire data containing exactly +one `0` or `1` per source bit. This matches the server cursor's reachable +positive-precision formatter rather than its unreachable character-formatting +branch. The implementation keeps one primitive long and its precision and +reuses the existing text sink. It adds no geohash object, lookup table, +converter registry, retained source value, server production change or per-row +string allocation. + +The packed overload accepts precision 1..60 and ignores higher input bits. The +textual overload preserves the existing public setter contract: strict, +case-insensitive base32 of 1..12 characters and five bits per character; null, +empty, overlong and invalid values fail locally. Omission is the only NULL +operation. All packed bit patterns are therefore values, including `0xff` at +eight bits. Schema-directed GEOHASH columns use the bitmap, so those +byte-aligned all-one values cannot collide with the fixed-width cursor sentinel. +Confirmed missing columns still infer the source precision. Legacy mode keeps +the original unflagged native GEOHASH frame and performs no DESCRIBE. + +The 15-row shared corpus covers precisions 1, 5, 7, 8, 15, 16, 20, 31, 32, 35 +and 60, leading zeroes, all-one values and masking of higher bits. Client and +server copies are byte-identical with SHA-256 +`8468be87f7061c58862b7bbe47290f508485c5cbc61a4373b2c3932e59a225e6`. +Component tests assert all 60 text widths, exact target wire bytes and bitmap +semantics, textual parsing, inference, duplicate suppression, rollback and +typed rejection of precision mismatch, unsupported, parameterized and +designated targets. Public socket tests cover exact schema frames, an existing +buffer pinned through a GEOHASH-to-VARCHAR rebind, and old-peer framing. Four +real-server tests cover missing-table inference, all shared text vectors, +exact native values including eight-bit `0xff`, A/error/C recovery, persisted +frame inspection and byte-preserving replay. + +The old `testCoercionToGeoHashErrors` expected every unsupported value to reach +the server. That no longer tests the product contract once schema mode rejects +the setter. It now asserts typed local errors and a subsequent successful row; +server-rejection tests remain separate instead of weakening a shared helper to +accept either path. This is a test-contract update, not a server behavior +change. + +On JDK 25, an expanded client regression passes 294 tests with zero failures, +errors or skips. It covers schema binding and Sender integration, table-buffer +and encoder behavior, legacy Sender framing, constants, earlier STRING-to- +GEOHASH, LONG256 and UUID conversions, recovered-frame analysis and replay. +The current client artifact installs successfully. The focused server gate +passes 15 tests: the four new GEOHASH tests, three STRING-to-GEOHASH tests, six +public Sender GEOHASH tests and two existing GEOHASH-to-text server tests. All +test JVM scratch roots are under `/mnt/pcie5`. + +The full `QwpSenderE2ETest` remains a diagnostic rather than an acceptance +gate: 137 tests produce two failures and 38 errors, improved from three failures +and 41 errors at D059. GEOHASH is no longer among them. The remaining product +gaps begin with integer temporal/text/decimal conversions, INT-to-IPv4 and +DOUBLE arrays; several other errors are legacy tests that still expect a +delayed server rejection after the new client correctly fails at the setter. +The next slice should implement integer temporal target representation, not +mechanically rewrite all remaining error tests. + +## D061 — Integer temporal targets keep raw target-unit counts + +Status: iteration 2.40 implemented and locally validated, 2026-09-14. Design +revision 63. The pre-iteration baseline is client `8932076f24` and server +`67e2c94f7a`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +BYTE, SHORT and INT setters now accept DATE, TIMESTAMP and TIMESTAMP_NS targets; +LONG also accepts DATE. This is target representation selection, not a unit +conversion. The client writes the integer unchanged as epoch milliseconds for +DATE, epoch microseconds for TIMESTAMP, or epoch nanoseconds for TIMESTAMP_NS. +`Integer.MIN_VALUE` and `Long.MIN_VALUE` retain their schema-mode source-null +meaning and become explicit bitmap NULLs. BYTE and SHORT have no source-null +sentinel. + +The implementation was cross-checked against `QwpWalAppender` and +`WalColumnarRowAppender`. The server admits BYTE, SHORT, INT and LONG into DATE, +and routes integer input for DATE and both timestamp types through the same raw +64-bit append. It performs no scale multiplication or division. The client +therefore adds DATE to the existing target-wire switch, adds the three temporal +targets to the existing small-integer helper, and adds DATE to the established +LONG path. No converter registry, temporal object, unit layer, retained value, +per-row allocation or server production change was added. Numeric identity +targets still short-circuit at the existing `isNumericTarget` check, so the new +temporal comparisons are not evaluated on that hot path; this is source-level +reasoning, not a new performance claim. + +The first component test draft incorrectly expected the timestamp raw-encoding +discriminator after a DATE column. Inspecting `QwpColumnWriter` showed that +DATE is a plain fixed-width value, while TIMESTAMP and TIMESTAMP_NS carry the +discriminator. The test was corrected; production code was unchanged. This +wire difference is why the implementation deliberately does not introduce a +shared temporal encoder. + +Both repositories consume the same 24-case corpus, SHA-256 +`0872eab8b83f608c2e12778e4e44512c87bb7cff7e693109aed7effacd42fd85`. +It covers BYTE and SHORT extrema for all three targets, INT null/negative/max +for all three targets, and LONG null/minimum-non-null/max for DATE. Client +component tests assert exact target wire types, bitmap state, timestamp +discriminators and raw values. Public scripted-Sender tests cover all four +setters, a failed partial row followed by a valid row, the one-refresh path, +parameterized DATE rejection, and a DATE block remaining pinned when the next +schema generation changes the target to TIMESTAMP_NS. The real-server test +writes the shared corpus through public Sender setters, waits for ACK, and +checks the stored raw values and nulls through SQL. + +Focused client validation passes 106 tests, and the expanded regression passes +297 tests with zero failures, errors or skips. Packaged-JAR verification passes +46 selected unit tests and both packaging integration tests. The focused server +gate passes the new corpus plus the existing TIMESTAMP and TIMESTAMP_NS +aggregate tests: three tests total, all green. All test JVM scratch roots are +under `/mnt/pcie5`. + +The released-binary matrix also passes all three combinations with the freshly +packaged client: current client to QuestDB 10.0.1 remains legacy, client 1.3.9 +to the current server remains legacy, and current client to current server uses +schema mode. The run used +`/mnt/pcie5/qwp-integer-temporal-compat/run.FKXeM7`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 37 errors, down from two failures and 38 errors at D060. The +existing TIMESTAMP aggregate is now green. The DATE aggregate still stops at +the separate STRING-to-DATE gap after its integer setters succeed, so it is not +claimed as this slice's acceptance test. Other remaining errors include integer +text/decimal targets, INT-to-IPv4, arrays and legacy tests whose expected +server-side rejection is now a local setter error. + +No setter-specific store-and-forward scenario was added. This slice changes +only construction of ordinary schema-pinned bytes; persisted framing and replay +remain opaque to column type. Exact public wire tests prove the new DATE and +timestamp bytes before publication, while the existing schema-frame replay +suite remains green. Adding a second recovery harness would duplicate those +contracts without exercising changed recovery code. + +## D062 — Small integers reuse the established text and symbol paths + +Status: iteration 2.41 implemented and locally validated, 2026-09-14. Design +revision 64. The pre-iteration baseline is client `853c3292c1` and server +`85f24785d3`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +BYTE, SHORT and INT setters now accept STRING, VARCHAR and SYMBOL targets in +schema mode. Each input widens losslessly to long and uses the existing reusable +numeric text sink. STRING and VARCHAR append direct VARCHAR wire data; SYMBOL +uses the existing global dictionary writer. `Integer.MIN_VALUE` retains its +source-null meaning and becomes an explicit target bitmap NULL. BYTE and SHORT +have no source-null sentinel. Missing columns still infer their native input +types, and a legacy peer still receives native BYTE, SHORT and INT wire columns +without a DESCRIBE. + +The implementation was cross-checked against the current server's +`QwpWalAppender` dispatch and `WalColumnarRowAppender`. Fixed-width integer +cursors targeting STRING, VARCHAR or SYMBOL all call `Numbers.append` on +`cursor.getLong()`. The client therefore extends the existing small-integer +target switch and reuses `formatLong`, `addString` and `addSymbol`. It adds no +source-width formatter, converter object, retained value, per-row `String`, +protocol field or server production change. Numeric identity targets still +short-circuit at the first `isNumericTarget` check, so the additional text check +is not evaluated on that hot path. This is source-level reasoning, not a new +performance claim. + +Client and server consume the same 30-case corpus, SHA-256 +`78481d27bbeb70354f309372650311e269ee365087e9003218222c2b0580dcfe`. +For each target it covers BYTE and SHORT extrema plus INT null, +minimum-non-null, negative one, zero, one and maximum. Component tests assert +the exact target wire type, UTF-8 payload or symbol dictionary entry, offsets, +IDs and bitmap null. Public socket tests additionally cover a rolled-back symbol +dictionary entry, A/error/C row recovery, the standard one-refresh path, a +SYMBOL block pinned before a VARCHAR schema generation, parameter rejection and +old-peer native framing. + +The real-server E2E uses the public `Sender` API and one flush across STRING, +VARCHAR and SYMBOL tables, waits for the acknowledged sequence, drains WAL and +asserts exact SQL values and nullness for all 30 vectors. A second E2E proves a +partially written row is discarded after an INT-to-UUID local error, completed +rows survive and the same symbol value is reused. The focused server gate also +runs the existing `testCoercionToSymbol`; all three tests pass. + +Expanded client validation passes 305 tests with zero failures, errors or +skips. Packaged-JAR verification passes 47 selected unit tests and both +packaging integration tests. The released-binary matrix passes current client +to QuestDB 10.0.1 in legacy mode, client 1.3.9 to current server in legacy mode, +and current client to current server in schema mode. All scratch roots are +under `/mnt/pcie5`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 36 errors, down from 37 errors at D061. The SYMBOL aggregate is now +green. The STRING and VARCHAR aggregates advance through the small-integer +inputs and stop at the separate CHAR-to-text gap; this slice does not claim +that unresolved contract. Other open implementation entries include +small-integer decimal targets, INT-to-IPv4, STRING-to-DATE and DOUBLE arrays. + +No new store-and-forward test was added. The changed code only chooses an +already supported target-native VARCHAR or SYMBOL representation before the +frame is persisted. Exact public wire tests prove those bytes, while existing +schema-frame replay and the established long-to-text persistence coverage remain +green. A setter-specific replay test would repeat unchanged persistence code. + +## D063 — Small integers share the exact integer-decimal tail + +Status: iteration 2.42 implemented and locally validated, 2026-09-14. Design +revision 65. The pre-iteration baseline is client `3acdceadbd` and server +`9bfa31a2fc`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +BYTE, SHORT and INT setters now accept DECIMAL8, DECIMAL16, DECIMAL32, +DECIMAL64, DECIMAL128 and DECIMAL256 targets in schema mode. Each non-null +source is an exact scale-zero integer. The binding passes its widened value to +one shared tail that applies the target's declared scale, checks its declared +precision and appends the matching target storage width. `Integer.MIN_VALUE` +retains its source-null meaning and becomes a scaled bitmap NULL. BYTE and SHORT +have no source-null sentinel. Missing columns still infer their native integer +types; legacy mode is unchanged. + +The current server source supports the same rule. `QwpWalAppender` dispatches +BYTE, SHORT, INT and LONG cursors to the small, 64-bit, 128-bit or 256-bit +decimal target appenders. `WalColumnarRowAppender` reads the integer through +`getLong()`, assigns scale zero, performs exact rescaling, validates precision +and storage, and writes the target decimal. The client therefore extracts the +existing LONG decimal branch into `appendIntegerDecimal` and calls it from all +four integer sources. Numeric and temporal target branches retain their earlier +short-circuits; no conversion registry, retained decimal object, per-value +allocation, protocol field or server production change was added. This is +source-level hot-path reasoning, not a new performance claim. + +Client and server consume the same 30-case corpus, SHA-256 +`e3ffe93367c2a6e65d1eeac821b3ccd798d88a9a4a7e37dfaaf9548a89d17ddf`. +It covers every one of the 18 source/target-width pairs, INT null at every +width, positive and negative scale conversion, source extrema and precision +overflow. Component tests assert the exact target wire type, scale byte, bitmap +and raw 64-bit limbs. Public socket tests additionally cover parameter rejection, +precision failure rollback, A/error/C row recovery and a DECIMAL(3,0) block +remaining pinned before a DECIMAL(20,5) schema generation. + +The real-server corpus test sends every valid vector through public Sender +setters, waits for the acknowledged sequence, drains WAL and asserts exact SQL +values and nulls. A second public-Sender test proves a partially written row is +discarded after an INT-to-UUID local error while completed decimal rows survive. +The focused server gate also runs the existing decimal aggregate and variant +tests: four tests total, all green. + +Expanded client validation passes 323 tests with zero failures, errors or skips. +Packaged-JAR verification passes 48 selected unit tests and both packaging +integration tests. The released-binary matrix passes current client to QuestDB +10.0.1 in legacy mode, client 1.3.9 to current server in legacy mode, and current +client to current server in schema mode. The matrix run used +`/mnt/pcie5/qwp-small-integer-decimal-compat/run.0W5Ji1`; all other scratch roots +also remain under `/mnt/pcie5`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 34 errors, down from 36 errors at D062. Both decimal aggregate +tests are now green. The open intentional implementation entries are +INT-to-IPv4, STRING-to-DATE and DOUBLE arrays. STRING/VARCHAR aggregates still +stop at the separately unresolved CHAR-to-text contract; other errors are +legacy tests whose expected server-side rejection is now an early local error. + +No new store-and-forward test was added. The changed code selects target-native +decimal bytes before persistence and reuses the already exercised decimal +buffer representation. Exact public wire tests prove those bytes; the existing +schema-frame replay suite remains green. A small-integer-specific recovery test +would repeat unchanged persistence code. + +## D064 — INT-to-IPv4 translates both source and target null sentinels + +Status: iteration 2.43 implemented and locally validated, 2026-09-14. Design +revision 66. The pre-iteration baseline is client `1630c04775` and server +`50d487e8b5`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +`intColumn` now accepts an IPv4 target in schema mode and emits target-native +`TYPE_IPv4`. `Integer.MIN_VALUE` is the INT source null sentinel and becomes an +explicit bitmap NULL. A supplied integer zero also becomes bitmap NULL because +zero is QuestDB's IPv4 null sentinel. Every other 32-bit pattern is appended +unchanged, including negative Java integers representing addresses whose first +octet is at least 128. BYTE, SHORT and LONG remain rejected for IPv4, and a +parameterized IPv4 target remains rejected. + +This rule is cross-checked against the current server's `QwpWalAppender` IPv4 +dispatch, `QwpFixedWidthColumnCursor` null detection and +`WalColumnarRowAppender.putIntToIPv4Column`. The legacy server path reads a +TYPE_INT cursor, translates its source-null row to IPv4 zero, and copies every +other integer verbatim. The client performs the equivalent conversion before +buffering, using the existing four-byte IPv4 column. The implementation adds +one allowed-target condition and one switch arm; it adds no parser, formatter, +address object, allocation, retained state, protocol field or server production +change. Other integer target arms are unchanged. This is source-level reasoning, +not a measured throughput claim. + +The component test covers `Integer.MIN_VALUE`, zero, one, minimum-plus-one, +maximum, all-one bits and omission. It asserts the exact TYPE_IPv4 wire code, +bitmap and four stored values. Negative tests retain BYTE/SHORT rejection and +cover IPv4 parameter rejection. Public socket tests prove A/error/C rollback, +the standard one-refresh lifecycle, exact target wire data, and a DECIMAL(3,0) +block remaining pinned before a new IPv4 generation. + +The real-server E2E uses public Sender calls for both null cases and six +non-null patterns from `0.0.0.1` through `255.255.255.255`, plus omission. It +waits for the ACK, drains WAL and checks exact rendered IPv4 values and marker +rows; a partial `failed-B` row is absent after an INT-to-UUID local error. The +focused gate also runs the existing INT-null IPv4 test: two tests total, both +green. Its stale comment was corrected because the current client now exercises +target-native schema encoding rather than the server's legacy TYPE_INT arm. + +Focused client validation passes 116 tests. Expanded client validation passes +326 tests, all with zero failures, errors or skips. Packaged-JAR verification +passes 49 selected unit tests and both packaging integration tests. The +released-binary matrix passes all three old/new combinations with current JAR +SHA-256 `82140012a88ee02b4aff5e7554a722f9f8a55b1eb83fb3763bd027a9cbc1469d`; +the run used `/mnt/pcie5/qwp-int-ipv4-compat/run.CKpGSZ`. + +The full `QwpSenderE2ETest` remains diagnostic: 137 tests now produce two +failures and 33 errors, down from 34 errors at D063. The INT-to-IPv4 test is now +green. The only intentional conversion implementations left are STRING-to-DATE +and DOUBLE arrays; the separately unresolved and legacy-expectation cases remain +outside this slice. + +No new store-and-forward test was added. The changed code selects the existing +target-native IPv4 representation before persistence. Exact public wire tests +prove those bytes, while the unchanged schema-frame replay suite remains green. +An INT-specific recovery test would repeat the already covered opaque-frame +persistence contract. + +## D065 — STRING-to-DATE mirrors the fixed server grammar before buffering + +Status: iteration 2.44 implemented and locally validated, 2026-09-14. Design +revision 67. The pre-iteration baseline is client `19557e2f` and server +`5338ac0845`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +`stringColumn` now accepts an ordinary DATE target in schema mode. It attempts +the same formats, in the same order, as the current server's +`DateFormatUtils.parseDate`: PostgreSQL date-time, date-only, date plus zone, +date-time with milliseconds plus zone, UTC date-time with exactly three +fractional digits, then a raw signed epoch-millisecond long. Successful values +are appended as target-native `TYPE_DATE`; `Long.MIN_VALUE` becomes an explicit +bitmap NULL. Missing columns still infer VARCHAR. A parameterized DATE and a +designated DATE remain rejected. + +The implementation extends the existing fixed timestamp parser with DATE's +five parse attempts and millisecond calendar/timezone arithmetic. It reuses the +same immutable JDK-derived timezone metadata and performs no per-value date +object allocation. It does not add a general format compiler, converter +registry, retained parser state, protocol field or server production change. +One precise guard rejects negative parsed years that wrap beyond a dynamic +zone's recurring-rule cutoff. Current server source otherwise reaches +`ConcurrentIntHashMap` with a negative year key and throws unchecked +`IllegalArgumentException`; schema mode must report a typed invalid value +instead of accepting bytes the legacy conversion cannot produce. This is a +bounded compatibility guard, not a general defensive layer. + +Client and server consume the same 53-case corpus, SHA-256 +`048c9360eb48eeda7785c7a0accc72ef3f87361ea9219fc7c6bddafa92f2f3a9`. +It covers raw long forms and the null sentinel, every grammar, numeric and +named zones, daylight-saving gaps and overlaps, hour 24, greedy fractional +milliseconds, year zero and negative years, wrapping arithmetic, invalid +calendar/time fields, missing or unknown zones, trailing input and Unicode +digits. A deterministic 100,015-input differential matches the current server +through its normal return/error domain. The server's unchecked extreme-year +named-zone crash is separately pinned as a local typed rejection. + +Component tests assert exact DATE wire type, bitmap and raw millisecond values +for the corpus, plus duplicate, omission, rollback, reset, inference and target +metadata errors. Public socket tests prove exact target wire and rollback, and +that a completed DATE block retains its schema snapshot before the next block +rebinds to VARCHAR. Real-server tests write every valid corpus value through +public Sender calls, wait for ACK, drain WAL and compare exact raw DATE values +and nullness; a second A/error/C test proves partial-row cancellation. The +existing successful DATE coercion test and its now-local public rejection test +also pass: four focused server tests total. + +Expanded client acceptance passes 443 tests with zero failures, errors or +skips. Packaged-JAR verification passes the three DATE component tests and both +packaging integration tests. The released-binary matrix passes current client +to QuestDB 10.0.1 in legacy mode, client 1.3.9 to current server in legacy mode, +and current client to current server in schema mode. Current artifact hashes +are client `58836b2fb469c87e264ff2d5f1a7f17f8e6ff9ad5ace3f6f0e5014034b44022d` +and server `dec115136d83b45f4f390bee7b50337fb0da7ee93cd6ae6c0427c0794fe31ce9d`; +the run used `/mnt/pcie5/qwp-string-date-compat/run.OpLD7e`. + +The full `QwpSenderE2ETest` remains diagnostic: its intentional conversion +error count falls from 33 to 32 because both DATE tests are now green. That run +also encountered two independent server `No space left` errors and produced +cascading assertion failures, so its unrelated failure count is not acceptance +evidence. The final intentional conversion implementation left is ranked +DOUBLE-array identity. BINARY parser paths, LONG_ARRAY, non-ASCII CHAR text and +malformed decimal metadata remain explicit server-contract decisions. + +## D066 — DOUBLE arrays reuse the existing wire path and pin rank + +Status: iteration 2.45 implemented and locally validated, 2026-09-14. Design +revision 68. The pre-iteration baseline is client `47a37a6f` and server +`c1f970220f`, whose submodule pins that exact client revision. Neither +repository is committed by this decision record. + +All four public `doubleArray` representations now participate in schema mode. +The primitive overloads have ranks 1, 2 and 3. The N-dimensional `DoubleArray` +wrapper exposes its current rank through an O(1) accessor, including after +`reshape`. A known target must be a canonical strong DOUBLE array of exactly +that rank. A confirmed missing column infers DOUBLE_ARRAY and pins its first +effective rank in the existing column buffer across resets; a later different +rank fails locally rather than creating a mixed-rank block. Null references +remain no-ops, empty shapes remain values, and the existing duplicate-first, +row rollback and one-refresh rules are preserved. Duplicate suppression runs +before inspecting a wrapper's rank, so an invalid or closed ignored second +value cannot change first-value-wins behavior. + +This is rank-checked identity, not an element conversion. The binding selects +the existing `TYPE_DOUBLE_ARRAY` column, and the established buffer and encoder +continue to own shape validation, element copying and wire output. No converter, +second shape model, per-row object, array extension parameter, send-time +transformation or server production change was added. The wrapper accessor +avoids a second traversal or temporary capture before validation. The only new +retained state is one integer on a confirmed-missing inferred column; known +schemas already carry rank in the encoded `ColumnType`. + +No throughput claim is made. There was no successful schema-aware array path +to compare before this slice, while the legacy append and encode loops are +unchanged. The new successful path adds only constant-time schema/rank checks +before the same element copy, so a synthetic A/B number would not isolate a +real regression boundary. + +The exclusion of LONG arrays is source-backed. The current server validates +DOUBLE_ARRAY batch rank but does not compare a cursor's element type with an +existing array target. `WalColumnarRowAppender.putArrayColumn` then constructs +storage from the cursor element type. Auto-creation rejects LONG_ARRAY, and SQL +does not currently create LONG[] columns. Accepting DOUBLE_ARRAY-to-LONG[] or +enabling `longArray` in schema mode would therefore reproduce a server +validation gap, not implement a supported cast. Synthetic schema tests pin the +local rejection because a real-server LONG[] fixture is not constructible. + +Five component tests cover every public representation, ranks 1, 2, 3 and 32, +live wrapper reshaping, all null overloads, empty shapes, raw NaN payload bits, +signed zero, infinity, duplicates, ragged inputs, rollback, reset, missing-rank +pinning, wrong element type, weak rank, parameters and designated-column +rejection. Three focused public-socket tests cover exact target wire bytes, +A/error/C rollback, the standard one-refresh path, generation pinning and null +no-op before negotiation. The expanded client acceptance selection passes 494 +tests with zero failures, errors or skips. Packaged-JAR verification passes the +five component tests and both packaging integration tests. + +The focused real-server gate passes nine tests: two new tests cover ranks 1 +through 4, all public representations, empty shapes, exact stored shapes, ACKs +and partial-row recovery; seven existing array/rejection tests now assert the +new early local errors. The complete 137-test `QwpSenderE2ETest` diagnostic has +113 passing tests, two assertion failures and 22 early schema errors. None is +an unimplemented intentional conversion: 17 are old aggregate rejection tests, +three are specialized tests that still expect a server-side error stage, two +are the explicit CHAR-to-text boundary, and the assertion failures are stale +empty/null-column-name message checks. With the test JVM started on +`/mnt/pcie5`, this run has no disk-space infrastructure failures. + +The released-binary matrix passes current client to QuestDB 10.0.1 in legacy +mode, client 1.3.9 to current server in legacy mode, and current client to +current server in schema mode. Current artifact hashes are client +`6d3cb13f4058e1cf9a2af7b3459266a2beffb4a92e7f148880dc5fcf40b24292` +and server +`87df2cee84f1cad1c450831fc2ec6f995437cb5eda701486bf452a8c3a3da524`; +the run used `/mnt/pcie5/qwp-double-array-compat-final/run.pK2tjd`. + +No array-specific store-and-forward test was added. Schema mode writes the +same self-contained DOUBLE_ARRAY bytes before persistence, and the 494-test +gate includes schema network replay, self-sufficient-frame and recovery replay +coverage. Repeating that unchanged persistence path with another value family +would not add a new contract assertion. + +A full client-module diagnostic encountered the existing +`QuestDBServerRecoveryTest.testFacadeStartsWhileServerDownThenWritesAndReaderConnectsOnRecovery` +expectation: the first effective write now waits for mandatory schema +negotiation and times out while no server exists. The timed-out test leaves its +reconnecting sender active, so that diagnostic run was stopped. This is a +separate startup-contract fixture, not an array regression; the 494-test +acceptance set is green. + +The intentional, server-supported public-setter conversion catalogue is now +complete. BINARY parser paths, LONG_ARRAY, non-ASCII CHAR text and malformed +decimal metadata remain explicit server-contract decisions, not unfinished +client converters. + +## D067 — ASYNC cold start uses best-effort legacy encoding + +Status: design accepted, implementation pending, 2026-09-15. Design revision +69. No production code or tests changed in this decision record. + +The full client-module diagnostic exposed a real contract conflict rather +than an array failure. `lazy_connect=true` selects the existing asynchronous +initial-connect policy, whose public contract says producers can buffer while +the server is unavailable. Schema activation instead made the first effective +write wait up to 30 seconds for negotiation and throw `SCHEMA_UNAVAILABLE`. +Preserving both mandatory cold-cache schema lookup and non-blocking offline +writes would require retaining original values for later conversion. That +would reintroduce the rejected send-time transformation model. + +Reuse the existing initial-connect choice instead. `OFF` makes one strict +foreground connection/negotiation attempt, while `SYNC` retries strictly within +its configured reconnect budget. Before any successful handshake, `ASYNC` +accepts a row immediately using the existing legacy conversion and flag-clear +framing, provided recovery has not already found schema-framed data. The first +effective operation pins the row's mode. A handshake completed during a partial +legacy row affects only the next row; buffered bytes are never relabelled, +rewritten or reconverted. + +This is a narrow startup exception, not a permanent schema switch. A successful +old-server handshake keeps subsequent rows legacy, but does not prohibit a +later one-way upgrade when another connection confirms schema support. Once +support is confirmed, cache misses and later outages retain the strict +30-second lookup/error contract and never fall back. A recovered extended +backlog also carries the sticky schema requirement before the fresh sender has +connected, so it overrides `ASYNC` fallback and waits for a supporting server. + +Source review confirms the simple wire path. The server rejects a schema flag +without negotiation but accepts flag-clear legacy frames on a negotiated +connection. The sender already pins a partial row and separates legacy and +schema-bound buffers at a row boundary. The cursor engine already records when +recovered frames require schema support. Implementation therefore needs no new +protocol field, raw-row store, deferred converter, persisted side file or +schema-specific configuration option. + +The availability cost is explicit. Pre-handshake rows use the whole legacy +conversion contract, not merely weaker validation. Handshake timing can change +successful stored values for cases such as sub-microsecond timestamps, +overflow and source-null normalization. A bad legacy row can also be accepted +locally and later receive a terminal NACK that stops the persisted queue; no +schema reply can repair its bytes. API documentation and release notes must +call this legacy conversion before confirmation and explain existing terminal +recovery, rather than promise deterministic schema-mode results. + +Required acceptance covers offline `ASYNC` startup followed by old and new +servers, handshake before the first row and during a partial row, old-to-new +upgrade, exact legacy/schema value differences, mixed replay across restart, +recovered schema-required backlog while offline, terminal rejection of a bad +startup row and a post-upgrade cache miss. Replace the stale facade recovery +expectation that a blocking first write returns before the test starts its +server; retain its observable construction, reconnect, write and read recovery +coverage under the new contract. + +## D068 — Integrate ASYNC fallback with one existing sticky latch + +Status: implemented and validated in the working tree, 2026-09-15. Design +revision 70. Not yet committed. + +Further source tracing reduced D067 to one producer-side decision in +`QwpWebSocketSender.bindingForEffectiveWrite()`. Preserve the existing partial- +row return first and call `ensureConnected()` as today. Before the current +bounded negotiation/lookup path, return the existing legacy binding result +when `initialConnectMode == ASYNC && !cursorEngine.requiresSchema()`, clearing +the per-call schema-freshness marker as on other legacy/in-progress paths. + +Do not consult `hasEverConnected()`. A fresh asynchronous sender and a sender +successfully connected to an old server both need legacy behavior, so +distinguishing them adds no decision value. The engine requirement already +becomes sticky when a supporting handshake succeeds and is restored directly +from recovered schema-framed data. It is volatile and published before the I/O +loop exposes the supporting connection, so it is also the correct race and +recovery authority. If it flips immediately after the producer reads false, +the current row is deliberately legacy and the existing row-boundary adoption +makes the next row schema-aware. + +No new state, helper abstraction, cache probe, schema mode, configuration key, +protocol field, raw-value owner, encoder path, persisted format, coordinator or +server change is justified. Existing retired-buffer and mixed-flush code owns +legacy-to-schema ordering, while the server already accepts flag-clear data on +a negotiated connection. + +The essential new coverage is narrow. Replace the asynchronous first-write +blocking test with one public test that buffers legacy data offline, connects +to a supporting server and verifies exact legacy then schema representations; +use a nanosecond `Instant` so the semantic difference is observable. Retarget +the existing timeout and interruption tests to a confirmed schema connection +whose describe response is withheld. Add one public recovered-schema-backlog +test proving that `ASYNC` still waits or fails instead of appending legacy data. +Reuse the existing facade late-server recovery, partial-row old-to-new upgrade, +mixed restart replay, real-server negotiated-legacy and terminal-NACK tests; +do not duplicate those fixtures. Final acceptance requires the complete client +suite and the standing released-binary compatibility matrix. + +The implementation matches the brief. `bindingForEffectiveWrite()` keeps its +in-progress-row path first, starts the existing I/O loop, then returns the +legacy binding for `ASYNC` only while the cursor engine does not require +schema. No production state, helper abstraction, coordinator path, encoder, +store-and-forward format, protocol field or server code changed. The public +`InitialConnectMode.ASYNC` documentation now states the legacy-before-support, +sticky-upgrade and recovered-backlog behavior. + +One fixture assumption was disproved during implementation: its `ts` column was +marked as the designated timestamp, so it could not expose the ordinary +timestamp representation difference. The fixture now reports no designated +column for that test. A nanosecond `Instant` then proves the actual contract: +the offline row contains legacy microseconds and the post-handshake row contains +schema-mode nanoseconds. + +Four focused new or changed tests pass. The full +`QwpSchemaSenderIntegrationTest` passes 72 tests and +`InitialConnectAsyncTest` passes seven. The complete client module passes 3,720 +tests with zero failures or errors and seven pre-existing skips. The current +server's five-test `QwpSchemaIdentityE2ETest` passes, including mixed +legacy/schema replay and legacy data on a negotiated connection. + +The released-binary matrix also passes: current client to QuestDB 10.0.1 uses +legacy conversion, client 1.3.9 to the current server remains legacy, and the +current client to current server enables schema mode. The packaged artifacts +used client SHA-256 +`74bbc706b69a53b89b84ce738865321b639d06b92200e1e1fa497e85c36e2e40` +and server SHA-256 +`38d597d80b12bb3c5ea13aa490b23df83061fb7299394b7c36a2c8227860627d`. +The completed full-client, build, server and compatibility runs used scratch +under `/mnt/pcie5`; `git diff --check` passes. + +## D069 — Restore CHAR-to-text compatibility with valid UTF-8 + +Status: implemented and locally validated in the working tree, 2026-09-15. +Design revision 71. Not yet committed. + +The complete server E2E classes exposed a conversion omitted by the earlier +hand-picked gates. On a known STRING or VARCHAR column, `charColumn` requested +only a CHAR target and therefore threw `UNSUPPORTED_FEATURE`. The existing +server accepts CHAR for both text targets, so mandatory schema mode turned a +previously valid public call into a local failure. The design had documented +the pair as an unresolved boundary, but that classification was not acceptable +for an always-on feature. + +Resolve the actual target in the existing `charColumn` path. Keep exact CHAR +storage unchanged. For STRING and VARCHAR, append the character through one +reused single-character sink into the existing target-native VARCHAR buffer. +This adds no conversion framework, protocol field, sender state, replay path, +server change or per-row allocation. + +Do not reproduce the server formatter's non-ASCII VARCHAR truncation. A valid +non-surrogate Java CHAR is encoded as valid UTF-8 for both text targets. A lone +UTF-16 surrogate cannot be represented as a standalone Unicode character, so +the setter reports `INVALID_VALUE` instead of silently replacing or corrupting +it. Native CHAR targets continue to preserve every 16-bit code unit. + +The focused component test passes four tests and checks exact target wire bytes +for ASCII and non-ASCII input plus surrogate rejection. The complete client +module passes 3,721 tests with no failures or errors and seven skips. After +installing that client artifact, the complete 138-test real-server E2E class +passes both existing regressions (`testCoercionToString` and +`testCoercionToVarchar`) and a new non-ASCII SQL assertion. Its remaining two +failures and 20 errors are the previously catalogued DATE/GEOHASH wire issues +and stale legacy rejection expectations, down from two failures and 22 errors +before this change. All test scratch owned by this work is under `/mnt/pcie5`. + +## D070 — Align complete server E2Es with mandatory schema validation + +Status: accepted, 2026-09-15. The associated QuestDB changes are test-only. + +The complete server test classes confirmed that the remaining failures were +stale test contracts and unstable test-only buffer handles, not missing client +conversions. Twenty-two Sender cases expected a server rejection for invalid +input that a schema-aware client must now reject before encoding. Their shared +helper now accepts only typed, non-retryable local schema errors and verifies +row recovery. A separate strict helper retains the server-NACK contract for +server-owned failures. No helper may accept either outcome interchangeably. + +The deferred-commit case now states the actual ownership rule: a local schema +error cancels only the current partial row. It cannot ask the server to roll +back already completed deferred rows because the invalid row never reached the +server. Empty column names, STRING input for DOUBLE, and timestamp overflow +likewise assert local structured errors and subsequent valid writes. + +DATE and GEOHASH integration fixtures exposed a separate test bug. They held a +test-only column object across rows, but schema installation can replace the +sender's active table buffer at the row boundary. Public integration tests now +write through the public row API. The six deliberately low-level tests keep +their raw coverage but reacquire the active column on every row. This adds no +production state, compatibility path, defensive check or abstraction. + +The two real-server asynchronous in-flight/fragmentation tests no longer try to +force a type mismatch through a schema-aware public API. They now send a valid +public row to a non-WAL table: local validation succeeds and the server returns +the server-owned terminal NACK whose propagation those tests exercise. The +immediate type-mismatch test was removed because that state is unreachable after +discovery and its useful NACK behavior duplicates the existing non-WAL test. +Deterministic client tests continue to cover broader asynchronous NACK and +terminal-error delivery. + +Validation passes the complete 138-test `QwpSenderE2ETest`, the complete +117-test `QwpWebSocketSenderReceiverTest` with one existing skip, and all six +`QwpSenderLowLevelTest` cases. The combined schema-aware server gate passes 480 +tests with zero failures or errors and one existing skip. The complete client +module passes 3,721 tests with zero failures or errors and seven existing skips. +The two asynchronous server-rejection cases pass three times each, including +their fragmented-transport variant. Final server scratch is under +`/mnt/pcie5/qwp-schema-aware-tests/final-repeat`; the final complete client run +uses `/mnt/pcie5/qwp-schema-aware-tests/client-final`, with no `/tmp` paths in +its 295 current Surefire reports. No client or server production source changed +in this decision. + +### Legacy-to-schema adoption moves to the batch boundary + +The 2026-09-17 complexity review ranked the duplicated mixed-family flush first. +Rather than fold it into the split path, this decision removes its cause. Row +boundary adoption was the only source of mixed legacy/schema batches. A batch +now has one wire contract, selected by its first committed row +(`isPendingBatchLegacy`); `bindingForEffectiveWrite()` keeps returning the +legacy contract while that batch is pending, and the first row after the flush +adopts schema mode. The upgrade stays one-way and is delayed by at most one +batch. `flushPendingRowsMixed()`, `mixedFramesFit()`, `encodeSingleTableFrame()` +and `isMixedSchemaBatch()` are deleted; the encoder is unchanged. This +supersedes the earlier "next effective row adopts schema mode" rule. The +trade-off: rows written after the handshake but inside the pending legacy batch +keep legacy conversion (for example microsecond `Instant` truncation). Which +rows preceded the handshake was already a timing race. + +The rewrite exposed an existing defect, reproduced on unmodified source: legacy +row, flush, supporting reconnect, then a row on the same table threw +`IllegalStateException: schema binding requires an empty, unbound table buffer` +from the column setter. `installSchemaBinding()` bound in place whenever the +buffer had no rows, but a flushed legacy buffer keeps its column layout. It now +binds in place only a buffer with no columns and replaces the buffer otherwise. +Both rewritten upgrade tests cover that path. + +### FLAG_SCHEMA moves from 0x02 to 0x40 + +Merging server `master` on 2026-09-18 surfaced a wire-bit collision that git +merged cleanly: server PR #7531 (browser negotiation) had already claimed `0x02` +for `FLAG_DURABLE_ACK_POLL`, a zero-table control frame. The ingress dispatch +read a poll frame as a schema-flagged frame on a connection that had not +negotiated schema and rejected it, failing seven durable-ack poll tests in +`QwpIngressAckLeapfrogTest` and `QwpIngressUpgradeProcessorResumeRecvTest`. +The poll bit is already on `master`, while `FLAG_SCHEMA` is unreleased, so +`FLAG_SCHEMA` moves to the free bit `0x40` in both the server and the client +`QwpConstants`. This supersedes the `FLAG_SCHEMA=0x02` value recorded in design +revision 13. The trade-off: store-and-forward frames persisted by an earlier +build of this branch carry `0x02` and no longer replay as schema frames. + +## OFF failover and per-stream replay requirements + +Schema negotiation no longer marks an engine as requiring schema-framed replay. +The engine latches that requirement from live or recovered frame content, before +publication, and retains it for its lifetime even after those frames are acknowledged. +Each reconnect factory carries only its own engine's requirement. OFF requests +schema support only to replay such a backlog; new rows remain legacy. + +AUTO chooses lookup eligibility from the installed connection and uses the cache +while disconnected. A pending lookup reaching a legacy replacement fails with +SCHEMA_UNAVAILABLE, allowing AUTO's existing fallback. STRICT separately remembers +a successfully installed schema connection, so a handshake alone still prevents +new legacy rows during an outage or downgrade. Neither policy memory nor another +stream's backlog rejects a legacy endpoint. Recovered schema backlog still waits +for a supporting connection before replay. diff --git a/design/schema-aware-sender-remaining-failures-plan.md b/design/schema-aware-sender-remaining-failures-plan.md new file mode 100644 index 000000000..60ee00155 --- /dev/null +++ b/design/schema-aware-sender-remaining-failures-plan.md @@ -0,0 +1,723 @@ +# Plan: resolve the remaining 47 schema-aware Sender test failures + +Status: all five phases accepted through iteration 2.36, 2026-09-12. The user +requested sequential completion of the remaining conversions. All original 47 +failures now pass. +Follow the [main design](schema-aware-sender.md); +decision history and evidence belong in the [journal](schema-aware-sender-journal.md). + +## Recommendation and scope + +Fix the nine stale expectations/fixtures first, then add conversions in small +end-to-end increments: text output, timestamps, small scalar targets, decimals. +Do not treat 47 failing methods as 47 production bugs, or implement a generic +conversion framework to make them disappear. + +All functional changes are in QWP client encoding/binding. No SQL parser, +planner or execution-engine change is proposed. SQL in these tests creates +tables and verifies stored results. + +| Phase | Baseline failures addressed | Count | Baseline failures left after the phase | +| --- | --- | ---: | ---: | +| 1 | Existing semantics and invalid test fixtures | 9 | 38 | +| 2 | LONG, UUID and floating-point text output | 8 | 30 | +| 3 | Ordinary timestamp input/output conversions | 4 | 26 | +| 4 | CHAR, LONG256 and GEOHASH targets | 4 | 22 | +| 5 | Decimal conversions | 22 | 0 | + +These are coverage checkpoints, not instructions to force the counts down. +A method is resolved only when its intended contract is tested and passes. +The substeps below are separate reviewable increments, not one large phase PR. + +Current checkpoint: phases 1–4 and phases 5a–5e are accepted. Phase 5a covers +all 18 logical native-decimal pairs; phase 5b covers LONG into all six decimal +target classes; phase 5c covers STRING into all six; phase 5d covers FLOAT and +DOUBLE into all six; phase 5e covers all native decimal widths into STRING and +VARCHAR. The original 56-test conversion class is green. Exact-current client +validation passes the 46-test public integration class, including four phase-5e +tests, plus two packaging checks, 950 canonical tests and an expanded 1,275-test +regression gate. Server validation passes three Phase-5e E2Es, all 56 original +conversion tests, 363 canonical tests +and three 126-test seeded repeats with 24 selected XML reports each. +Legacy mode, row recovery and pinned encodings are preserved. +D041 records the NaN-null approval and formatter work; D042 records the small +raw timestamp extension, corrected wire fixtures and native LONG measurements. +D043 records the fixed timestamp parser, its source-quirk regressions, broad +matched-runtime differential and the measured 2–3% native string-write cost. +D044 records checked timestamp-to-text normalization, exact server formatting, +63 shared vectors and 160,202 public differential checks. Its native timestamp +performance guard detects no repeatable regression. +D045 records CHAR's first-character conversion, exact wire-presence versus +SQL-null distinction, 23 shared vectors, 65,549 public differential checks and +the missing standard schema-refresh catch caught by the new setter's own test. +D046 records STRING-to-LONG256's four-local-limb parser, 23 shared vectors, +40,055 public differential checks, exact rollback/generation wire tests and +independent legacy acceptance/rejection. The native string-write regression +guard passes; it is not a LONG256 throughput measurement. +D047 records all 60 GEOHASH precisions, exact schema bitmap framing, empty/null +normalization, 7,210 public differential checks and canonical packed-type +validation. The two-path performance guard passes with measured small costs: +about 1.46–1.80% append time and 0.17–1.15% encoding time in native string checks. +D048 records phase 5a's exact rescale/precision/storage checks, target-scale +retention, caller immutability, MISSING/legacy behavior, row rollback/schema +generation and bitmap-only decimal null characterization. Its 60-row shared +corpus has 45 values and 15 invalid cases across all 18 logical pairs (SHA-256 +`3819694736f54de4d2f2f396b09ff487667051c6045b34cc6d7579552551d11d`). +The frozen candidate JAR SHA-256 is +`1e814a2f82836b5c9ac04cf7b5674e56f1b939440df486ae6628ebb1b8eebb08`. +The three seeded 101-test repeats have 20 archived XML reports each. +D049 records phase 5b's scale-zero LONG conversion, stable schema-mode +`Long.MIN_VALUE` null, target-scale locking, direct target-width wire output, +rollback and schema-generation coverage. Its shared corpus has 39 rows: 19 +values, 14 invalid values and six null cases across every decimal target class +(SHA-256 +`d9a47432a14583f8675b0035ab10c3ef9f36e78c9f4aab6a8634958eb5aafcb1`). +The frozen candidate JAR SHA-256 is +`41d137a1c2388ed21f776e372771785377d91c15a887d94f04e82c97f505d403`. +The 12-run native LONG-to-LONG guard detects no repeatable 5% cycles or +instructions regression; it is not decimal-throughput evidence. +D050 records phase 5c's target-precision/scale parser, direct target wire output, +stable bitmap-null rules and deliberate correction of three unsafe server +behaviors. Its 55-row shared corpus has 19 values, 24 invalid values and 12 +nulls (SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`). +The installed candidate JAR SHA-256 is +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +The 3,601,944-case differential passes ordinary server parity and records 72 +deliberate malformed-prefix corrections. The 12-run native STRING-to-VARCHAR +guard detects no repeatable 5% cycles or instructions regression; it is not +decimal, network or ingestion throughput evidence. +D051 records phase 5d's exact FLOAT widening, shortest round-tripping decimal, +lossless target-scale check, non-finite bitmap nulls and direct target-width +output. Its shared corpus has 60 rows: 37 values, 17 invalid values and six nulls +(SHA-256 +`366a889719c6e7b688ff00e1bdfb95e273ab192508a1094dac5312c3d899611d`). +The 5,996,868-case differential matches the server's acceptance, target scale and +limbs. The installed candidate JAR SHA-256 is +`da44a24c6d100a0bff8a984f37c40409a0e3cc3f132f7ac9304a9e0b5d25c99d`. +Performance validation caught and removed a native hot-inlining regression by +keeping decimal conversion in the existing cold nonnumeric helper. The final +12-run guards detect no material native FLOAT/DOUBLE regression; they do not +measure decimal conversion, network or ingestion throughput. + +D053 records phase 5e's corrected Decimal256 promotion, unsigned scale-byte +semantics, full-physical public values and intentional per-value mixed-width/ +scale behavior for known text targets. Its 24-row shared corpus has 18 values +and six public null cases across all six source/target pairs (SHA-256 +`4fd43e6d1c08fc76cee6505e6ef06a250aa30281bd686bb0115ba6c710065b5f`). +The 3,000,294-case differential matches server scalar formatting over full +physical limbs and arbitrary integer scales. The accepted and installed JAR is +`e06759605ee3d1cf01c3277ba13531b0172e42138042c1f2f49c2db49e04b22d`. +Six counterbalanced pairs for each native identity width show instruction +changes of +1.05%, +1.38% and +0.71%, with no repeatable 5% regression. This is +an identity-path guard, not a decimal-to-text, network or ingestion benchmark. + +Baseline: iteration 2.20, client HEAD +`981bdb02a471f3b290c89b8e78cbc422610e329e`, server HEAD +`12a33d651e51e2682e7a448c8db5168fc72dfad3`, with the recorded dirty working +trees. Frozen client JAR SHA-256: +`102128206d124f8963b89dd6f6764da641086697272f0bb9d93493c55903330b`. + +Server evidence is in +`core/target/schema-text-source-null.2.20-server/run-manifest.md`, +`logs/broad.log` and `logs/broad-failure-names.txt`: the fixed 333-test +selection has seven failures and 40 errors, exactly 47 methods. The separate +271-test green gate and three 26-test seeded repeats passed. The inventory at +the end assigns every failing method exactly once. + +## Phase 1: restore honest tests — 9 failures + +### 1a. Separate existing schema and legacy behavior — 5 + +Accepted in iteration 2.21. The following describes the completed repair. + +The three boolean tests already exercise working client conversions but expect +legacy omissions to become false/zero. Update the schema expectations to the +target's missing value: null for nullable targets, zero for BYTE/SHORT. Assert +nullness as well as printed values. Keep the existing independent unflagged +legacy comparison in `QwpSchemaBooleanE2ETest`. + +The two floating tests deliberately describe legacy behavior: supplied NaN can +depend on the bitmap, and positive 2^63 can clamp to LONG maximum. Their current +public Sender calls instead apply the already-approved schema rules. Send the +legacy cases as actual unflagged FLOAT/DOUBLE frames, preserving both the +accepted values and server rejection checks. Keep explicit schema tests for +NaN normalization and local rejection at 2^63 in +`QwpSchemaFloatingNumericE2ETest`; do not restore legacy behavior in production. + +### 1b. Remove direct mutation of Sender-owned buffers — 4 + +Accepted in iteration 2.22. The following describes the completed repair. + +The two geohash-to-text and two designated-LONG timestamp tests obtain a buffer +through `getTableBuffer()` and manually add source columns. Subsequent public +`at()`/`atNow()` installs schema bindings and uses extended framing, so these +are no longer valid fixtures. + +Rebuild them with a standalone native buffer, ordinary unflagged encoder and +nonrequesting WebSocket client. Assert source types, geohash precision, bitmap, +values, ACK and exact stored rows. Keep named LONG designated-timestamp wire +semantics separate from public `at()`/unit conversion. Do not relax Sender +ownership or add a schema opt-out. + +The public native geohash converter remains unimplemented after this repair. +Keep that fact visible; a passing legacy wire test does not implement it. + +## Phase 2: text output — 8 failures, three increments + +### 2a. LONG to STRING/VARCHAR/SYMBOL — 3 + +Accepted in iteration 2.23. The following describes the implemented contract. + +Approved on 2026-09-11 and recorded in the main contract before code: +supplied `Long.MIN_VALUE` becomes a target null, as an effective write rather +than a no-op. Ordinary values use canonical signed decimal text. The legacy +bitmap-present special value is the four-character text `"null"`, not decimal +minimum; preserve the 2.20 raw characterization. + +Use the existing target VARCHAR representation for STRING/VARCHAR and existing +dictionary-backed SYMBOL representation for SYMBOL. Reuse verified formatters +and append paths, with no raw-LONG fallback. Cover MIN+1/MAX, zero, negatives, +omission, explicit sentinel, repeated symbols and duplicate-first behavior. + +### 2b. UUID to STRING/VARCHAR — 2 + +Accepted in iteration 2.24, following separate approval on 2026-09-11. Only the pair +with both limbs equal to MIN becomes target SQL NULL, as an effective write. +A single minimum limb remains a real value. Preserve `(lo, hi)` input order and +canonical lowercase formatting. Do not add UUID-to-SYMBOL or change native UUID +bytes, inference or legacy behavior. The verified checkpoint is 35 to 33, with +only the two UUID-to-text baseline failures removed and no additions. + +The obsolete public UUID-to-text rejection cases in +`QwpSchemaTextSourceNullE2ETest` now exercise unsupported UUID-to-SYMBOL instead; +its independent legacy-byte tests are unchanged. Shared literal vectors, +null-first ignored duplicates, omission/reset, public row recovery, native/text +encoding generations and component-seeded text replay are covered. + +### 2c. FLOAT/DOUBLE to STRING/VARCHAR/SYMBOL — 3 baseline methods + +Accepted in iteration 2.25. The following describes the implemented contract. + +Cover the floating input family coherently, including the FLOAT overload +although these three failing methods happen to use DOUBLE. Match the server's +actual formatter, including FLOAT widening before formatting. Check signed zero, +finite rounding boundaries, infinities and NaN with/without an omitted row. + +Separately approved on 2026-09-11: supplied FLOAT/DOUBLE NaN becomes SQL NULL +for all three text/SYMBOL targets, as an effective first write. Legacy raw +characterization confirms bitmap-present NaN becomes literal `NaN` instead. +Do not substitute Java formatting or add a per-row policy switch. Differential +tests found that the older client formatter differs from current server Ryu; +keep source-faithful formatting isolated from legacy paths. + +## Phase 3: ordinary timestamp conversions — 4 failures + +### 3a. LONG to timestamp — 1 + +Accepted in iteration 2.26. The following describes the implemented contract. + +Reuse target timestamp encoding. LONG denotes a raw count in the target unit, +not a value whose unit is inferred or converted from micros. Test both micro- +and nanosecond targets, negative/zero/boundary values and source-null behavior. +This must not enable named writes to the designated timestamp. + +### 3b. STRING to timestamp — 1 + +Accepted in iteration 2.27. The following describes the implemented contract. + +Port/reuse the server's actual timestamp grammar and value checks, not an SQL +cast or an assumed ISO-only parser. The server uses its microsecond parser +and checked multiplication by 1000 for a nano target. Test accepted/rejected +grammar, overflow, fractional precision, null/omission and failed-row recovery. + +Use a fixed parser for the eleven actual formats, preserving their priority, +not the server's general compiler. Copy the relevant calendar helpers exactly: +`yearMicros` has asymmetric negative-overflow saturation, while surrounding +arithmetic and timezone subtraction can wrap. The narrow zone adapter must +retain locale-token ordering and use the separately parsed year for future +rules, even after epoch arithmetic wraps. Plain JDK instant lookup differs on +a reproduced extreme-year input (D043). Use public JDK metadata, not reflection +or a general timezone engine. Same-runtime differential tests must precede +acceptance; runtime locale/tzdb data and the class-initialized reference century +remain independently sourced by client and server. Do not quietly narrow the +grammar or add protocol metadata to this increment. + +### 3c. Timestamp to STRING/VARCHAR — 2 + +Accepted in iteration 2.28. The following describes the tested contract. + +Cover both primitive timestamp units and `Instant`, not only the MICROS calls +in the failing tests. The extra overloads are coherent-family acceptance +coverage, not additional failures in the 47-method count. +The server formats text to millisecond precision; nanoseconds first +divide by 1000 with truncation toward zero to obtain micros, then +`MicrosFormatUtils.appendDateTime` uses the millisecond UTC pattern. Characterize +negative sub-microsecond values, extremes and source-null behavior. Existing +timestamp-target range/precision exceptions do not silently extend to text. +The timestamp cursor uses bitmap nulls only: present MICRO MIN formats as empty +text, while present NANO MIN formats as a finite historical timestamp. Preserve +these semantics; the LONG/NaN text-null approvals do not apply by analogy. +Separately approved by the user: checked conversion into the legacy source +precision for larger units and Instant, rejecting out-of-range values before +append. Keep legacy mode unchanged. D044 records this text-target overflow +decision; it is not an implicit extension of D021. + +## Phase 4: small scalar targets — 4 failures + +### 4a. CHAR identity and STRING to CHAR — 2 + +Accepted in iteration 2.29. The following describes the completed slice. + +Use target-native CHAR encoding and the existing native CHAR setter. For text +input, reproduce QWP's first decodable BMP-character behavior. Empty input, +malformed raw UTF-8 and a supplementary first code point produce NUL. Test +public malformed UTF-16 separately: `stringColumn` applies replacement during +UTF-8 encoding, so assert the actual replacement bytes and resulting CHAR, +not an assumed NUL. Test more than the three ASCII values in the old test. + +Do not bundle CHAR-to-text: its non-ASCII VARCHAR behavior is a separate, +documented server compatibility issue. + +### 4b. STRING to LONG256 — 1 + +Accepted in iteration 2.30. The following describes the completed slice. + +The target LONG256 representation and server-equivalent parser/limb order use +existing value storage. Tests cover accepted syntax, invalid/overflow input, +null, leading zeros, boundaries, exact wire bytes and failed-row recovery. + +Read-only source verification after 2.29 confirms a lowercase `0x` prefix and +2–64 hex digits in complete byte pairs; hex digits themselves are case-insensitive. +Reject signs, whitespace, odd digit counts and overlong input. Parse right-to-left +into four local longs and append with existing addLong256; no retained scratch +object is needed. Java null/omission is null, but the literal whose four limbs +are all Long.MIN_VALUE must be rejected, matching the server parser/appender. + +Native LONG256 setter activation remains separate, unimplemented work. + +### 4c. STRING to GEOHASH — 1 + +Accepted in iteration 2.31. The following describes the completed slice. + +Bind target bits from the schema and encode the parsed value with that precision. +Match the server's base32 parsing, truncation, empty/null and consumed-character +rules, including its 12-character cap. Test multiple precisions and invalid or +insufficient input, not just GEOHASH(4c). + +Use the precision already packed in the server's column type. The current +schema response sends that full type code and an empty extension block; +decimal precision/scale is likewise already encoded in its type code. Do not +add new protocol parameter fields or stop rejecting unknown extension blocks +just to enable these types. Keep precision pinned with the existing schema +snapshot. Native geohash input/text-output activation remains separately tracked; +the phase-1 raw fixture repair does not supply it. + +Read-only preparation after 2.30 identifies one storage detail to address: +`addGeoHash` currently sets precision only on a value write, but a null-only +column must also carry the schema's precision. Reuse the existing precision +field and keep the adjustment narrow. + +Source revalidation for 2.31 corrects the initial empty-value proposal. Native +GEOHASH without a bitmap treats a packed all-ones value as null, losing valid +maximum values at byte-aligned precisions. With a bitmap, packing empty text +as all ones can instead produce a non-null value when packed width is smaller +than storage width. Preserve the server's VARCHAR conversion result: normalize +empty text to bitmap null, and always include the bitmap for schema-directed +GEOHASH columns, including all-present batches. This costs one bit per row and +needs no value scan or new retained state. Force only columns already using +nullable storage; low-level non-nullable schema buffers and legacy encoding +remain unchanged. Runtime characterization and exact wire tests establish both +cases: source-wire presence does not imply target-wire presence. Recognize only +the complete canonical packed type for a valid precision, preserving rejection +of malformed and future-flagged types. + +## Phase 5: decimals — 22 failures, five increments + +Keep decimals last because target precision/scale, source scale, wire width and +null semantics are shared dependencies. Reuse existing decimal arithmetic and +8/16/32-byte column storage only after checking their behavior against the server. + +| Increment | Baseline methods | Required behavior | +| --- | ---: | --- | +| 5a. Native decimal to decimal (accepted in 2.32) | 4 | Target precision/scale and wire representation; all source widths; exact rescale, narrowing and overflow | +| 5b. LONG to decimal (accepted in 2.33) | 4 | Checked scaling and target precision/storage limits; explicit source-null rule | +| 5c. STRING to decimal (accepted in 2.34) | 4 | Server-equivalent grammar and target-scale parsing; precision-loss/overflow failures | +| 5d. FLOAT/DOUBLE to decimal (accepted in 2.35) | 4 | Actual server rounding/conversion behavior, non-finite values, underflow and overflow | +| 5e. Native decimal to text (accepted in 2.36) | 6 | All three input widths to STRING/VARCHAR, preserving per-value scale and server scalar formatting | + +5a is accepted in iteration 2.32 and produces a working real-Sender E2E before +the later input families are added. Small decimal targets use DECIMAL64 wire +storage while enforcing the narrower target precision and scale. Convert +every input width/scale to the target precision, scale and compatible storage +width at setter time. The encoded decimal scale must therefore equal the +target scale; the original input scale is only conversion input. Assert that +wire scale and exact converted limbs in cross-width E2Es. Do not merely relabel +source limbs or leave rescaling/precision validation to the server. Reuse +existing append storage by supplying already-converted values, rather than +adopting its legacy first-input-scale rule for schema bindings. +Cover emitted all-null columns separately from null setters that omit the +column entirely; neither has a non-null coefficient to rescale. + +Read-only preparation after phase 4c finds an existing per-column Decimal256 +scratch value. Reuse it for copy/rescale/validation before append rather than +adding another binding-owned scratch object. Preserve caller-owned input values. +Keep internal high-to-low decimal words separate from the writer's low-to-high +wire order. Also preserve the schema's target scale across reset and failed-row +cleanup: a following batch may emit that retained column entirely as nulls, +without invoking a decimal setter. Legacy columns must keep their current +first-input-scale-per-batch behavior. Prove both paths before choosing the +smallest metadata-lifetime change; do not share a new blanket reset policy. + +The accepted implementation covers all 18 logical pairs with nine physical +source-width/target-width paths. It copies each caller value into the existing +per-column Decimal256 scratch, performs exact rescale plus target-precision and +storage-width checks, then appends the direct target wire width. Target scale +survives reset and failed-row cleanup. Caller objects are unchanged; a failed +partial row rolls back without damaging completed rows or schema-generation +boundaries. Confirmed MISSING and legacy paths retain source-width inference, +public null/no-op behavior and the legacy first-input-scale rule. + +Decimal cursors use bitmap-only nullness, not the fixed-width sentinel rule. +Characterize raw sentinel limbs with and without a bitmap and preserve public +typed-decimal null/no-op behavior. Check representable source scales and +representative malformed scale bytes; if an existing server validation gap is +found, report it separately rather than copying unsafe acceptance into the +client or quietly broadening this plan into a server rewrite. + +That characterization is complete for 5a: explicit bitmap nulls are null, while +raw sentinel limbs without a null bitmap remain supplied conversion input at the +server boundary. At the 2.32 checkpoint this did not activate decimal-to-text +or the later input families. + +5b is accepted in iteration 2.33. For a known decimal target, LONG starts as an +exact scale-zero integer in the existing per-column Decimal256 scratch, then +uses 5a's rescale, precision/storage validation and target-width append tail. +No conversion registry, second scratch value or Sender/protocol state was added. +Small decimal targets still use DECIMAL64 wire storage while enforcing their +declared precision. MISSING still infers LONG. + +The schema-mode source-null rule is now explicit: `Long.MIN_VALUE` becomes a +decimal bitmap null and locks the target scale before append. Its meaning is +therefore independent of other rows' omissions. Legacy encoding and server +conversion are unchanged and retain the existing bitmap-dependent result, +which a raw-frame real-server test characterizes separately. Duplicate +suppression happens before validation, and an invalid partial row is rolled +back while completed rows remain. A schema-generation test proves that a +sealed DECIMAL64(scale 4) block retains its bytes when a refresh selects +DECIMAL128(scale 5) for the later block. + +The 39-row shared client/server corpus covers all six target classes and has +19 accepted values, 14 local rejections and six source-null cases. Component +tests assert exact type, bitmap, scale and low-to-high limbs; public Sender E2Es +assert exact stored values and recovery. The original four server conversion +methods fail against the frozen 5a client and pass against 5b. After acceptance, +the unchanged original diagnostic has 14 errors, all assigned to 5c–5e. + +The legacy-path performance guard passes 12 balanced A/B runs: instructions +change by +0.001% to +0.002% and paired cycles by -0.47% to +1.99%. A separate +candidate-only nine-physical-pair characterization has six runs of 20 million +rows each (180 million conversions and 3.36 GB verified per run), with basic +medians of 32.91 ns, 182.03 cycles and 1140.18 instructions per value. It has no +matched schema baseline and makes no network, ingestion or general-throughput +claim. One 200-million-row sizing run and an interrupted follow-on are excluded. + +5c is accepted in iteration 2.34. For a known decimal target, STRING is parsed +at the target precision and scale with `strict=false` and `lossy=false`, including +the server grammar's exponent underscores. The existing Decimal256 scratch then +goes through phase 5a's precision/storage checks and direct target-width append +tail. The shared parser uses the current server's leading-zero and full-scale +precision rule; a schema-only parser fork was not added. Small decimal targets +continue to use DECIMAL64 wire storage while enforcing their declared precision +and scale. + +Java null and exact optionally signed `NaN` or `Infinity`, after the parser's +supported suffix processing, become target-scale bitmap SQL nulls for all six +target classes. Malformed special-value prefixes and extreme nonzero integer- +boundary exponents fail locally as `INVALID_VALUE`. Schema mode also avoids the +server's small-decimal special-value narrowing bug by emitting bitmap nulls. +The exact-special and exponent checks live in the shared parser, so direct +`Decimal64`, `Decimal128` and `Decimal256` string parsing changes globally: it +rejects malformed special prefixes and reports extreme nonzero exponents as +`NumericException`. MISSING and legacy Sender paths remain VARCHAR and retain +server-side conversion. + +The implementation adds no state, framework, protocol, server production, SQL +or send-time transformation. The 55-row shared corpus contains 19 values, 24 +invalid inputs and 12 nulls and has SHA-256 +`67739711b320b07fd4d5eb72ecd19e14b5999f6a07e756ed83a847d809577568`. +Client red validation failed eight tests before implementation. Final focused +validation passes eight tests plus two packaging checks, all-decimal validation +passes 463 tests, and the canonical gate passes 941. The candidate and installed +JAR SHA-256 is +`30e593b27a6eb71c022e198bb5e331af973af643e1fd3eedde8555c8d2d2112c`. +The 3,601,944-case differential passes ordinary server parity and identifies 72 +intentional malformed-prefix corrections. Synchronizing shared-parser precision +metadata corrected 19 stale assertions in 16 existing tests without changing +their parsed values or scales. + +Server focused validation passes seven tests and the canonical gate passes 352. +The original 56-test conversion class has ten errors: only the four +STRING-to-decimal methods disappeared, with no additions. Three seeded repeats pass 115 tests and +22 archived XML reports each. The 12 balanced fixed-work native +STRING-to-VARCHAR runs share checksum `15640344951909530213`; paired basic +instructions change by -0.001% to -0.000% and cycles by -1.028% to -0.007%, +with no repeatable 5% regression. This makes no decimal, network or ingestion +throughput claim. + +5d is accepted in iteration 2.35. FLOAT is widened exactly to DOUBLE, the existing +schema Ryu formatter emits the shortest round-tripping decimal, and the existing +decimal parser applies the target precision and scale with `lossy=false`. +Natural-scale and precision overflow therefore reject instead of rounding or +truncating. NaN and both infinities become target-scale bitmap nulls, matching all +six server decimal paths. This is intentionally not the SQL cast contract. + +The implementation reuses the phase-5a Decimal256 scratch, validation and direct +width append tail. It adds no new parser, state or conversion framework. The +first branch placement increased native FLOAT/DOUBLE instructions by about +3.8–4.7% after pushing a common method over the hot inlining threshold. Moving +decimal dispatch into the existing cold nonnumeric helper restores the common +method's prior bytecode shape. Six counterbalanced pairs per native path then +show median instruction changes of +0.0001% for FLOAT and +0.0037% for DOUBLE. + +The 60-row shared corpus covers every source/target-class pair, exact width, +scale and limbs, no-rounding cases, non-finite nulls, signed zero, underflow and +overflow. Client and public Sender tests cover duplicate-first-wins, omission, +invalid-row rollback and a DECIMAL64-to-DECIMAL128 schema generation change. +Four real-server E2Es consume the same corpus and independently preserve raw +unflagged legacy FLOAT/DOUBLE bytes. The unchanged 56-test diagnostic now leaves +only the six 5e errors. + +5e is deliberately binding-only. For a known STRING or VARCHAR target, promote +the supplied value into one lazily reused binding-owned Decimal256, reduce its +scale to the unsigned low QWP byte, format that promoted value into the existing +reused text sink, and append `TYPE_VARCHAR`. This mirrors the server's scalar +conversion and avoids the narrow formatters' precision limit for full physical +Decimal64/128 values. Preserve valid source scales, trailing zeros and plain +notation. Public Java null and decimal `NULL_VALUE` remain setter no-ops; they +do not claim first-value-wins. MISSING and legacy mode keep native decimal tags +and the original source scale. Do not add a formatter abstraction, conversion +registry or table-buffer state. + +Scalar parity is not legacy batch emulation. Once schema binding establishes a +text target, every setter value is converted independently, so mixed native +decimal widths and scales in one target column are supported. Legacy inference +locks one native width and scale for a column block and may reject or rescale a +later value. Recreating that irrelevant intermediate constraint would add state +and make text conversion depend on neighboring rows, contrary to the schema- +directed design. Cover this intentional behavior difference explicitly. + +The shared corpus must cover all six source/target pairs, each physical width, +positive, negative and zero coefficients, maximum precision/scale, leading and +trailing fractional zeros, and both public null forms. Public socket tests assert +the exact `TYPE_VARCHAR` bitmap, offsets and bytes while covering null-then-value, +duplicate-first-wins before conversion, A/failed-B/C rollback, omission, scratch +reuse and a DECIMAL64-to-VARCHAR schema-generation change. A second component +test would repeat the same binding path without adding an observable contract. +Real-server tests run the same valid corpus and an independent unflagged native- +decimal frame so the old server formatter remains covered. + +Public low-level constructors and `setScale` mutators can expose coefficients +or scales outside the documented 18/38/76-digit domains. Phase 5e does not add +a local rejection policy: Decimal256 promotion preserves the physical value and +the unsigned low scale byte matches what the server receives. Public sentinel +values remain setter no-ops, so non-bitmap sentinel formatting stays confined +to independent raw-frame characterization. Any Decimal API or server +rejection/normalization is separate work. + +## Acceptance required for every increment + +1. Read the entire affected test, its public setter path, target dispatch and + concrete parser/formatter overloads. Confirm the failure from the frozen + baseline; save pre-edit copies of every file to be changed. Record any + deliberate compatibility exception in the main design before code. +2. Implement only the selected family in the existing binding/buffer ownership. + No second schema cache, conversion registry, new connection state, opt-out, + send-time transformation, or source-type fallback. Avoid unrelated guards + and hot-path refactors; check shared-path allocation/throughput regressions + with comparable fixed work when those paths change. Do not claim a speedup + from source inspection or test timings. +3. Add a public-wire test proving direct target types and parameters. Add a + separate component test only when it covers a distinct observable contract. + Use shared, identical conversion vectors for client/server tests where + applicable. A server SQL result alone cannot prove client conversion. +4. Keep a real public-Sender E2E passing in that same increment. Cover completed + row A, partial invalid row B, valid row C; automatic rollback without + cancellation/reselection; first-value-wins; null vs omission; boundary values; + typed local errors; schema change/rebinding and replay where affected. + Pin auto-flush when asserting buffered-row preservation and await the + published ACK when assertions depend on feedback. +5. Preserve independent legacy native-wire acceptance/rejection tests and + byte-identical replay. A test helper must not accept either a local or a + server error interchangeably. SQL is only setup/result verification. +6. Run client component/public-Sender regressions for production changes, then + build/install one standalone client artifact. Freeze its hash before server + validation; no overlapping client install and server tests, no + `-Plocal-client`. Use an absolute project-local temp directory in the + Surefire JVM startup `argLine`, not merely a later system property. +7. Extend the current 271-test green server gate with the increment's tests and + recovered baseline cases. Rerun the unchanged 333-test diagnostic and compare + exact failing methods, not only totals. Run three seeded repeats of affected + integration/recovery cases; preserve failing development logs. +8. Independent SOL source/adversarial review plus architect acceptance. Update + this ledger, the main conversion inventory and the journal with source/JAR + hashes, commands, actual results and any next blocker. + +If a resolved first error reveals a later stale assertion, classify that +assertion against the approved contract. Do not stop at the first green setter, +delete the test, loosen its expected error, or silently count a renamed/removed +method as fixed. Readiness requires every baseline case's behavior to remain +represented. + +## Post-ledger whole-class cleanup — 2026-09-15 + +Running the complete server Sender classes after the 47-method ledger exposed +two kinds of test debt, not more missing conversions: + +- Twenty-two `QwpSenderE2ETest` cases still expected the server to reject input + that mandatory schema mode now rejects locally. They now require a typed, + non-retryable local schema error and cancel the partial row. The separate + helper for server-owned failures accepts only a server NACK. +- DATE and GEOHASH fixtures retained a test-only column handle across rows even + though schema discovery may replace the active table buffer after a row. + Public API tests now use the public row API. The six intentionally low-level + tests remain low-level and reacquire the active column for every row. + +The two asynchronous in-flight/fragmentation tests now use a public write to a +non-WAL table: schema validation succeeds, then the server returns the real +server-owned NACK those tests are meant to propagate. The immediate type- +mismatch test was removed because that path is no longer reachable after +discovery and its useful server-NACK behavior duplicates the existing non-WAL +test. Deterministic client tests retain broader terminal-error coverage. This is +a test-contract migration only: no client or server production code changed. + +The complete `QwpSenderE2ETest` passes 138 tests, the complete +`QwpWebSocketSenderReceiverTest` passes 117 with one existing skip, and the +complete `QwpSenderLowLevelTest` passes six. The combined schema-aware server +gate passes 480 tests with zero failures or errors and one existing skip. The +complete client module passes 3,721 tests with zero failures or errors and seven +existing skips. The two asynchronous server-rejection cases pass three times +each, including their fragmented-transport variant. + +## Definition of done and remaining release work + +Done for this plan: all 47 baseline cases are represented and pass, the original +333 selection is green (or has an explicitly recorded one-to-one test rename), +the accumulated green gate/client regressions pass, and independent legacy +checks retain their intended behavior. No unresolved new regression is accepted. + +This is not the complete feature-release checklist. The design still inventories +unrepresented BYTE/SHORT/INT families, DATE targets, IPv4, arrays, native geohash/ +LONG256 paths and other pairs, plus compatibility decisions for unsafe server +paths. Actual older-server-distribution testing, lookup-timeout configuration, +full lifecycle/compatibility validation and representative performance evidence +also remain release work. Do not advertise full schema conversion coverage just +because these 47 methods turn green. + +## Exact 47-method ledger + +Source files: +[conversion E2Es](../../questdb/core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpWebSocketTypeConversionE2ETest.java), +[Sender E2Es](../../questdb/core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpSenderE2ETest.java). +Group IDs match the increments above. All 47 entries are accepted. No case has +been removed or renamed in this original selection. + +### 1a. Existing schema/legacy expectations — 5 accepted + +- `QwpWebSocketTypeConversionE2ETest.testBooleanToNumericColumns` +- `QwpWebSocketTypeConversionE2ETest.testBooleanToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testBooleanToVarcharColumn` +- `QwpSenderE2ETest.testFloatingPositiveTwoToThe63ClampsToLongMax` +- `QwpSenderE2ETest.testFloatingToIntegerNaNDependsOnBlockNullBitmap` + +### 1b. Direct-buffer legacy fixtures — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testGeoHashToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testGeoHashToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToDesignatedTimestampMicroColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToDesignatedTimestampNanoColumn` + +### 2a. LONG to text/SYMBOL — 3 accepted + +- `QwpWebSocketTypeConversionE2ETest.testLongToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testLongToSymbolColumn` + +### 2b. UUID to text — 2 accepted + +- `QwpWebSocketTypeConversionE2ETest.testUuidToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testUuidToVarcharColumn` + +### 2c. Floating point to text/SYMBOL — 3 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDoubleToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToSymbolColumn` + +### 3a. LONG to ordinary timestamps — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testLongToTimestampColumn` + +### 3b. STRING to ordinary timestamps — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToTimestampColumn` + +### 3c. Timestamp to text — 2 accepted + +- `QwpWebSocketTypeConversionE2ETest.testTimestampToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testTimestampToVarcharColumn` + +### 4a. CHAR target — 2 accepted + +- `QwpWebSocketTypeConversionE2ETest.testCharToCharColumn` +- `QwpWebSocketTypeConversionE2ETest.testStringToCharColumn` + +### 4b. STRING to LONG256 — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToLong256Column` + +### 4c. STRING to GEOHASH — 1 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToGeoHashColumn` + +### 5a. Native decimal to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDecimalToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToSmallDecimalColumn` + +Residual P3: a scale-up overflow has the correct `INVALID_VALUE` reason and row +rollback, but its detail currently says `decimal value cannot be rescaled +exactly` instead of identifying overflow. Keep this as diagnostic wording debt; +do not add another hot-path branch solely to refine the message. + +### 5b. LONG to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testLongToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testLongToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testLongToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testLongToSmallDecimalColumn` + +### 5c. STRING to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testStringToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testStringToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testStringToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testStringToSmallDecimalColumn` + +### 5d. Floating point to decimal — 4 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDoubleToDecimal64Column` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToDecimal128Column` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToDecimal256Column` +- `QwpWebSocketTypeConversionE2ETest.testDoubleToSmallDecimalColumn` + +### 5e. Native decimal to text — 6 accepted + +- `QwpWebSocketTypeConversionE2ETest.testDecimalToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimalToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal128ToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal128ToVarcharColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal256ToStringColumn` +- `QwpWebSocketTypeConversionE2ETest.testDecimal256ToVarcharColumn`