From 656fa46bfabb06f1c305a99edae3216f233d78dd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 19:10:26 +0100 Subject: [PATCH 01/11] perf(qwp): preallocate arrays during result batch decoding --- .../src/_qwp/_core/result-batch.ts | 69 ++++++++----------- test/qwp/egress.test.ts | 49 +++++++++++++ 2 files changed, 79 insertions(+), 39 deletions(-) diff --git a/packages/client-core/src/_qwp/_core/result-batch.ts b/packages/client-core/src/_qwp/_core/result-batch.ts index c3869628..7db07251 100644 --- a/packages/client-core/src/_qwp/_core/result-batch.ts +++ b/packages/client-core/src/_qwp/_core/result-batch.ts @@ -1230,21 +1230,26 @@ function decodeGorillaValues(reader: QwpByteReader, count: number): bigint[] { return values; } +// Array.from({ length }, mapper) takes the generic array-like path; a +// preallocated array with an indexed loop avoids that overhead while still +// reading values in order. +function fillArray(count: number, mapper: (index: number) => T): T[] { + const values = new Array(count); + for (let index = 0; index < count; index++) values[index] = mapper(index); + return values; +} + function readTimestampValues( reader: QwpByteReader, count: number, gorilla: boolean, ): bigint[] { if (!gorilla) { - return Array.from({ length: count }, () => - reader.readBigInt64("timestamp value"), - ); + return fillArray(count, () => reader.readBigInt64("timestamp value")); } const encoding = reader.readUint8("timestamp encoding"); if (encoding === 0) { - return Array.from({ length: count }, () => - reader.readBigInt64("timestamp value"), - ); + return fillArray(count, () => reader.readBigInt64("timestamp value")); } if (encoding !== 1) { throw new QwpProtocolError(`unknown timestamp encoding: ${encoding}`); @@ -1285,14 +1290,14 @@ function readArrayValue( if (type === QWP_COLUMN_TYPE.DOUBLE_ARRAY) { return { dimensions: shape, - values: Array.from({ length: elementCount }, () => + values: fillArray(elementCount, () => reader.readFloat64("double array element"), ), }; } return { dimensions: shape, - values: Array.from({ length: elementCount }, () => + values: fillArray(elementCount, () => reader.readBigInt64("long array element"), ), }; @@ -1464,7 +1469,7 @@ export class QwpResultBatchDecoder { QWP_MAX_COLUMNS_PER_TABLE, "result column count", ); - this.schema = Array.from({ length: columnCount }, () => { + this.schema = fillArray(columnCount, () => { const nameLength = readCount( reader, QWP_MAX_IDENTIFIER_BYTES, @@ -1825,47 +1830,35 @@ export class QwpResultBatchDecoder { switch (schema.type) { case QWP_COLUMN_TYPE.BOOLEAN: { const bytes = reader.readBytes(Math.ceil(count / 8), "boolean values"); - dense = Array.from( - { length: count }, - (_, index) => (bytes[index >>> 3] & (1 << (index & 7))) !== 0, + dense = fillArray( + count, + (index) => (bytes[index >>> 3] & (1 << (index & 7))) !== 0, ); break; } case QWP_COLUMN_TYPE.BYTE: - dense = Array.from({ length: count }, () => - reader.readInt8("byte value"), - ); + dense = fillArray(count, () => reader.readInt8("byte value")); break; case QWP_COLUMN_TYPE.SHORT: - dense = Array.from({ length: count }, () => - reader.readInt16("short value"), - ); + dense = fillArray(count, () => reader.readInt16("short value")); break; case QWP_COLUMN_TYPE.CHAR: - dense = Array.from({ length: count }, () => + dense = fillArray(count, () => String.fromCharCode(reader.readUint16("char value")), ); break; case QWP_COLUMN_TYPE.INT: case QWP_COLUMN_TYPE.IPV4: - dense = Array.from({ length: count }, () => - reader.readInt32("int value"), - ); + dense = fillArray(count, () => reader.readInt32("int value")); break; case QWP_COLUMN_TYPE.FLOAT: - dense = Array.from({ length: count }, () => - reader.readFloat32("float value"), - ); + dense = fillArray(count, () => reader.readFloat32("float value")); break; case QWP_COLUMN_TYPE.DOUBLE: - dense = Array.from({ length: count }, () => - reader.readFloat64("double value"), - ); + dense = fillArray(count, () => reader.readFloat64("double value")); break; case QWP_COLUMN_TYPE.LONG: - dense = Array.from({ length: count }, () => - reader.readBigInt64("long value"), - ); + dense = fillArray(count, () => reader.readBigInt64("long value")); break; case QWP_COLUMN_TYPE.DATE: case QWP_COLUMN_TYPE.TIMESTAMP: @@ -1886,13 +1879,13 @@ export class QwpResultBatchDecoder { dense = this.readSymbols(reader, count, rowCount, deltaMode); break; case QWP_COLUMN_TYPE.UUID: - dense = Array.from({ length: count }, () => ({ + dense = fillArray(count, () => ({ low: reader.readBigUint64("UUID low bits"), high: reader.readBigUint64("UUID high bits"), })); break; case QWP_COLUMN_TYPE.LONG256: - dense = Array.from({ length: count }, () => ({ + dense = fillArray(count, () => ({ words: [ reader.readBigInt64("LONG256 word 0"), reader.readBigInt64("LONG256 word 1"), @@ -1911,7 +1904,7 @@ export class QwpResultBatchDecoder { : schema.type === QWP_COLUMN_TYPE.DECIMAL128 ? 16 : 32; - dense = Array.from({ length: count }, () => ({ + dense = fillArray(count, () => ({ unscaled: readSignedLittleEndian(reader, bytes, "decimal value"), scale: scale!, })); @@ -1925,7 +1918,7 @@ export class QwpResultBatchDecoder { ); } const byteCount = Math.ceil(precisionBits / 8); - dense = Array.from({ length: count }, () => { + dense = fillArray(count, () => { const bytes = reader.readBytes(byteCount, "geohash value"); return { bits: geohashLittleEndianValue(bytes, 0, precisionBits!), @@ -1936,9 +1929,7 @@ export class QwpResultBatchDecoder { } case QWP_COLUMN_TYPE.DOUBLE_ARRAY: case QWP_COLUMN_TYPE.LONG_ARRAY: - dense = Array.from({ length: count }, () => - readArrayValue(reader, schema.type), - ); + dense = fillArray(count, () => readArrayValue(reader, schema.type)); break; default: throw new QwpProtocolError( @@ -2012,7 +2003,7 @@ export class QwpResultBatchDecoder { } dictionary = local; } - return Array.from({ length: count }, () => { + return fillArray(count, () => { const id = readCount(reader, dictionary.length, "symbol ID"); if (id >= dictionary.length) { throw new QwpProtocolError(`symbol ID out of range: ${id}`); diff --git a/test/qwp/egress.test.ts b/test/qwp/egress.test.ts index eee7baab..65b94f64 100644 --- a/test/qwp/egress.test.ts +++ b/test/qwp/egress.test.ts @@ -512,6 +512,55 @@ describe("QWP result batch decoder", () => { ]); }); + it("keeps materialized values in wire order across nulls and nested arrays", () => { + const payload = new QwpByteWriter(); + payload.writeUint8(QWP_EGRESS_MESSAGE.RESULT_BATCH).writeBigUint64(0n); + writeQwpVarint(payload, 0); // batch sequence + writeQwpVarint(payload, 0); // table name + writeQwpVarint(payload, 4); // rows + writeQwpVarint(payload, 4); // columns + for (const [name, type] of [ + ["bool", QWP_COLUMN_TYPE.BOOLEAN], + ["ts", QWP_COLUMN_TYPE.TIMESTAMP], + ["array", QWP_COLUMN_TYPE.LONG_ARRAY], + ["sym", QWP_COLUMN_TYPE.SYMBOL], + ] as const) { + writeString(payload, name); + payload.writeUint8(type); + } + + payload.writeUint8(0).writeUint8(0b1010); // bit-packed booleans + payload.writeUint8(1).writeUint8(0b0010); // timestamp row 1 is null + payload.writeBigInt64(100n).writeBigInt64(200n).writeBigInt64(300n); + payload.writeUint8(0); // no array nulls + for (const values of [[1n, 2n], [], [3n], [4n, 5n, 6n]]) { + payload.writeUint8(1).writeInt32(values.length); + for (const value of values) payload.writeBigInt64(value); + } + payload.writeUint8(0); // local symbol dictionary + writeQwpVarint(payload, 2); + writeString(payload, "alpha"); + writeString(payload, "beta"); + for (const id of [1, 0, 1, 0]) writeQwpVarint(payload, id); + + const message = decodeQwpEgressMessage( + encodeQwpFrame(payload.toUint8Array(), 0, 1), + ); + if (message.kind !== "result-batch") throw new Error("unexpected message"); + const batch = new QwpResultBatchDecoder().decode(message); + expect(batch.columns.map((column) => column.values)).toEqual([ + [false, true, false, true], + [100n, null, 200n, 300n], + [ + { dimensions: [2], values: [1n, 2n] }, + { dimensions: [0], values: [] }, + { dimensions: [1], values: [3n] }, + { dimensions: [3], values: [4n, 5n, 6n] }, + ], + ["beta", "alpha", "beta", "alpha"], + ]); + }); + it("decodes identifiers at the defensive egress byte bound", () => { // Query results may expose existing Java metadata created through another // protocol. Keep accepting up to 127 UTF-16 code units on egress, while From acced673e6a4346e0e7142495d71acc658327f25 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 21:17:24 +0100 Subject: [PATCH 02/11] perf(qwp): use indexed fill in result view materialize() --- packages/client-core/src/_qwp/_core/result-batch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client-core/src/_qwp/_core/result-batch.ts b/packages/client-core/src/_qwp/_core/result-batch.ts index 7db07251..b7681023 100644 --- a/packages/client-core/src/_qwp/_core/result-batch.ts +++ b/packages/client-core/src/_qwp/_core/result-batch.ts @@ -908,7 +908,7 @@ export class QwpResultBatchView { this.columnViews.map((column) => ({ name: column.name, type: column.type, - values: Array.from({ length: this._rowCount }, (_, row) => { + values: fillArray(this._rowCount, (row) => { const value = column.get(row); // Binary values are zero-copy slices in the view API. materialize() // promises independently owned data, so detach those slices here. From 04ba50ec9c4255de7bf54c6369594f5966c75aec Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 22:07:58 +0100 Subject: [PATCH 03/11] perf(qwp): keep unboxed elements for numeric result arrays fillArray's single `new Array` allocation site is shared with schema, bigint, and object callers, so V8 pretransitions it to generic elements. DOUBLE_ARRAY values and materialize()d FLOAT/DOUBLE columns then stored every double as a boxed HeapNumber: ~24 instead of 8 bytes per element retained, and ~0.7x speed for consumer loops, versus Array.from on main. Give arrays handed to users that hold only numbers their own allocation sites: readFloat64Values for DOUBLE_ARRAY elements, and fillIntegers / fillFloats for null-free numeric columns in materialize(). --- .../src/_qwp/_core/result-batch.ts | 68 ++++++++++++++++--- 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/packages/client-core/src/_qwp/_core/result-batch.ts b/packages/client-core/src/_qwp/_core/result-batch.ts index b7681023..2c617afd 100644 --- a/packages/client-core/src/_qwp/_core/result-batch.ts +++ b/packages/client-core/src/_qwp/_core/result-batch.ts @@ -908,12 +908,20 @@ export class QwpResultBatchView { this.columnViews.map((column) => ({ name: column.name, type: column.type, - values: fillArray(this._rowCount, (row) => { - const value = column.get(row); - // Binary values are zero-copy slices in the view API. materialize() - // promises independently owned data, so detach those slices here. - return value instanceof Uint8Array ? value.slice() : value; - }), + values: + column.nonNullCount === this._rowCount && + INTEGER_COLUMN_TYPES.has(column.type) + ? fillIntegers(this._rowCount, (row) => column.get(row) as number) + : column.nonNullCount === this._rowCount && + FLOAT_COLUMN_TYPES.has(column.type) + ? fillFloats(this._rowCount, (row) => column.get(row) as number) + : fillArray(this._rowCount, (row) => { + const value = column.get(row); + // Binary values are zero-copy slices in the view API. + // materialize() promises independently owned data, so + // detach those slices here. + return value instanceof Uint8Array ? value.slice() : value; + }), ...(column.scale === undefined ? {} : { scale: column.scale }), ...(column.precisionBits === undefined ? {} @@ -1239,6 +1247,50 @@ function fillArray(count: number, mapper: (index: number) => T): T[] { return values; } +// The helpers below duplicate fillArray on purpose. V8 tracks an elements kind +// per allocation site: fillArray's `new Array` is shared with schema, bigint, +// and object callers, so V8 pretransitions it to generic elements, and every +// double stored there becomes a boxed HeapNumber (~24 instead of 8 bytes per +// element, and slower consumer loops). Arrays handed to users that hold only +// numbers get their own sites so they keep unboxed SMI/double elements. Do +// not fold these back into fillArray. +function readFloat64Values(reader: QwpByteReader, count: number): number[] { + const values = new Array(count); + for (let index = 0; index < count; index++) { + values[index] = reader.readFloat64("double array element"); + } + return values; +} + +const INTEGER_COLUMN_TYPES: ReadonlySet = new Set([ + QWP_COLUMN_TYPE.BYTE, + QWP_COLUMN_TYPE.SHORT, + QWP_COLUMN_TYPE.INT, + QWP_COLUMN_TYPE.IPV4, +]); +const FLOAT_COLUMN_TYPES: ReadonlySet = new Set([ + QWP_COLUMN_TYPE.FLOAT, + QWP_COLUMN_TYPE.DOUBLE, +]); + +function fillIntegers( + count: number, + mapper: (index: number) => number, +): number[] { + const values = new Array(count); + for (let index = 0; index < count; index++) values[index] = mapper(index); + return values; +} + +function fillFloats( + count: number, + mapper: (index: number) => number, +): number[] { + const values = new Array(count); + for (let index = 0; index < count; index++) values[index] = mapper(index); + return values; +} + function readTimestampValues( reader: QwpByteReader, count: number, @@ -1290,9 +1342,7 @@ function readArrayValue( if (type === QWP_COLUMN_TYPE.DOUBLE_ARRAY) { return { dimensions: shape, - values: fillArray(elementCount, () => - reader.readFloat64("double array element"), - ), + values: readFloat64Values(reader, elementCount), }; } return { From 6a52b5c4dad0a65cfaf2fd5d391ecbcce1d844dc Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 22:11:22 +0100 Subject: [PATCH 04/11] test(qwp): label SYMBOL null flag in wire-order test --- test/qwp/egress.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/qwp/egress.test.ts b/test/qwp/egress.test.ts index 65b94f64..9f0bcec2 100644 --- a/test/qwp/egress.test.ts +++ b/test/qwp/egress.test.ts @@ -537,7 +537,7 @@ describe("QWP result batch decoder", () => { payload.writeUint8(1).writeInt32(values.length); for (const value of values) payload.writeBigInt64(value); } - payload.writeUint8(0); // local symbol dictionary + payload.writeUint8(0); // no SYMBOL nulls; local dictionary follows writeQwpVarint(payload, 2); writeString(payload, "alpha"); writeString(payload, "beta"); From 886e8cad06b47a2f6adee65f3331f28485d8ea36 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 22:37:02 +0100 Subject: [PATCH 05/11] perf(qwp): skip BigInt varints and null expansion in result decoding readCount() and readQwpVarintNumber() built every count, length, and symbol ID as a BigInt only to convert it back to a number. The new internal readQwpVarintSmall() accumulates encodings of up to 7 bytes (49 bits, always exact) in a plain number and falls back to BigInt for 8-10-byte encodings, with the same validation, errors, and reader advancement as readQwpVarint(). It lives outside the _core barrel, so the public surface is unchanged; readQwpVarint() still returns bigint. For a column whose null flag is 0, readNullLayout() no longer allocates a rowCount-long boolean array and expandNulls() hands the fresh dense array over instead of copying it. Those arrays now reach users as-is, so null-free numeric columns are built through the number-only allocation sites (fillIntegers/fillFloats), which keeps FLOAT/DOUBLE query() values unboxed: 8 instead of ~24 bytes per element. Measured on a 10k-row batch of 2 SYMBOL, 10 LONG, 10 DOUBLE, and a TIMESTAMP column (medians of 5 processes): query() 2.33M -> 5.93M rows/s, batch.rows() 1.93M -> 2.55M, queryViews() 16.1M -> 56.5M; with 10% nulls +16%, +10%, and +85%. --- .../src/_qwp/_core/result-batch.ts | 51 ++++++++++----- .../src/_qwp/_core/varint-number.ts | 36 +++++++++++ packages/client-core/src/_qwp/_core/varint.ts | 4 +- test/qwp/core.test.ts | 62 +++++++++++++++++++ 4 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 packages/client-core/src/_qwp/_core/varint-number.ts diff --git a/packages/client-core/src/_qwp/_core/result-batch.ts b/packages/client-core/src/_qwp/_core/result-batch.ts index 2c617afd..29bd543e 100644 --- a/packages/client-core/src/_qwp/_core/result-batch.ts +++ b/packages/client-core/src/_qwp/_core/result-batch.ts @@ -13,7 +13,7 @@ import { } from "./constants"; import { QwpResultBatchMessage } from "./egress"; import { QwpProtocolError } from "./errors"; -import { readQwpVarint } from "./varint"; +import { readQwpVarintSmall } from "./varint-number"; import { decompressQwpZstdFrame } from "./zstd"; // One axis bound for both directions: a decoder that accepted an axis the @@ -1070,7 +1070,8 @@ function signedLittleEndianValue( } interface NullLayout { - nulls: boolean[]; + /** Per-row null flags, or null when the column carries no nulls. */ + nulls: boolean[] | null; nonNullCount: number; } @@ -1108,7 +1109,13 @@ function readCount( maximum: number, label: string, ): number { - const value = readQwpVarint(reader); + const value = readQwpVarintSmall(reader); + if (typeof value === "number") { + if (value > maximum) { + throw new QwpProtocolError(`${label} out of range: ${value}`); + } + return value; + } if (value > BigInt(maximum)) { throw new QwpProtocolError(`${label} out of range: ${value}`); } @@ -1120,8 +1127,8 @@ function readNullLayout(reader: QwpByteReader, rowCount: number): NullLayout { if (flag !== 0 && flag !== 1) { throw new QwpProtocolError(`invalid column null flag: ${flag}`); } + if (flag === 0) return { nulls: null, nonNullCount: rowCount }; const nulls = new Array(rowCount).fill(false); - if (flag === 0) return { nulls, nonNullCount: rowCount }; const bitmap = reader.readBytes( Math.ceil(rowCount / 8), @@ -1137,14 +1144,18 @@ function readNullLayout(reader: QwpByteReader, rowCount: number): NullLayout { return { nulls, nonNullCount }; } -function expandNulls( - dense: readonly T[], +function expandNulls( + dense: QwpResultValue[], layout: NullLayout, ): QwpResultValue[] { - const values = new Array(layout.nulls.length); + const nulls = layout.nulls; + // Every dense producer returns a fresh array of exactly nonNullCount + // elements, so a null-free column can hand it over without a copy. + if (nulls === null) return dense; + const values = new Array(nulls.length); let denseIndex = 0; - for (let row = 0; row < layout.nulls.length; row++) { - values[row] = layout.nulls[row] ? null : dense[denseIndex++]; + for (let row = 0; row < nulls.length; row++) { + values[row] = nulls[row] ? null : dense[denseIndex++]; } return values; } @@ -1252,8 +1263,10 @@ function fillArray(count: number, mapper: (index: number) => T): T[] { // and object callers, so V8 pretransitions it to generic elements, and every // double stored there becomes a boxed HeapNumber (~24 instead of 8 bytes per // element, and slower consumer loops). Arrays handed to users that hold only -// numbers get their own sites so they keep unboxed SMI/double elements. Do -// not fold these back into fillArray. +// numbers -- DOUBLE_ARRAY elements, null-free numeric columns from decode() +// and materialize() -- get their own sites so they keep unboxed SMI/double +// elements. Do not fold these back into fillArray, and never store null or a +// non-number through them. function readFloat64Values(reader: QwpByteReader, count: number): number[] { const values = new Array(count); for (let index = 0; index < count; index++) { @@ -1873,6 +1886,12 @@ export class QwpResultBatchDecoder { ): QwpResultColumn { const layout = readNullLayout(reader, rowCount); const count = layout.nonNullCount; + // Null-free dense arrays become the column values as-is, so numeric ones + // use number-only allocation sites to keep unboxed elements. Columns with + // nulls are copied into a generic array by expandNulls(), where boxed + // input is cheaper to copy. + const fillInts = layout.nulls === null ? fillIntegers : fillArray; + const fillDoubles = layout.nulls === null ? fillFloats : fillArray; let dense: QwpResultValue[]; let scale: number | undefined; let precisionBits: number | undefined; @@ -1887,10 +1906,10 @@ export class QwpResultBatchDecoder { break; } case QWP_COLUMN_TYPE.BYTE: - dense = fillArray(count, () => reader.readInt8("byte value")); + dense = fillInts(count, () => reader.readInt8("byte value")); break; case QWP_COLUMN_TYPE.SHORT: - dense = fillArray(count, () => reader.readInt16("short value")); + dense = fillInts(count, () => reader.readInt16("short value")); break; case QWP_COLUMN_TYPE.CHAR: dense = fillArray(count, () => @@ -1899,13 +1918,13 @@ export class QwpResultBatchDecoder { break; case QWP_COLUMN_TYPE.INT: case QWP_COLUMN_TYPE.IPV4: - dense = fillArray(count, () => reader.readInt32("int value")); + dense = fillInts(count, () => reader.readInt32("int value")); break; case QWP_COLUMN_TYPE.FLOAT: - dense = fillArray(count, () => reader.readFloat32("float value")); + dense = fillDoubles(count, () => reader.readFloat32("float value")); break; case QWP_COLUMN_TYPE.DOUBLE: - dense = fillArray(count, () => reader.readFloat64("double value")); + dense = fillDoubles(count, () => reader.readFloat64("double value")); break; case QWP_COLUMN_TYPE.LONG: dense = fillArray(count, () => reader.readBigInt64("long value")); diff --git a/packages/client-core/src/_qwp/_core/varint-number.ts b/packages/client-core/src/_qwp/_core/varint-number.ts new file mode 100644 index 00000000..aeca8f5c --- /dev/null +++ b/packages/client-core/src/_qwp/_core/varint-number.ts @@ -0,0 +1,36 @@ +import { QwpByteReader } from "./bytes"; +import { QwpProtocolError } from "./errors"; + +/** + * @internal Not re-exported from the `_core` barrel. + * + * Reads an unsigned LEB128 uint64 with the exact validation and reader + * advancement of readQwpVarint(), but accumulates encodings of up to 7 bytes + * (49 bits, always exact) in a plain number and returns it without touching + * BigInt. Counts, lengths, and symbol IDs take this path on every cell, so + * building each one as a BigInt only to convert it back dominated decoding. + * 8..10-byte encodings, including zero-padded small values, fall back to + * BigInt and return a bigint. + */ +export function readQwpVarintSmall(reader: QwpByteReader): number | bigint { + let byte = reader.readUint8("varint"); + if ((byte & 0x80) === 0) return byte; + let value = byte & 0x7f; + let scale = 0x80; + for (let index = 1; index < 7; index++) { + byte = reader.readUint8("varint"); + value += (byte & 0x7f) * scale; + if ((byte & 0x80) === 0) return value; + scale *= 0x80; + } + let big = BigInt(value); + for (let index = 7; index < 10; index++) { + byte = reader.readUint8("varint"); + if (index === 9 && (byte & 0xfe) !== 0) { + throw new QwpProtocolError("QWP varint exceeds uint64 range"); + } + big |= BigInt(byte & 0x7f) << BigInt(index * 7); + if ((byte & 0x80) === 0) return big; + } + throw new QwpProtocolError("QWP varint exceeds 10 bytes"); +} diff --git a/packages/client-core/src/_qwp/_core/varint.ts b/packages/client-core/src/_qwp/_core/varint.ts index bb73d3fa..8bc4d766 100644 --- a/packages/client-core/src/_qwp/_core/varint.ts +++ b/packages/client-core/src/_qwp/_core/varint.ts @@ -1,5 +1,6 @@ import { QwpByteReader, QwpByteWriter } from "./bytes"; import { QwpProtocolError } from "./errors"; +import { readQwpVarintSmall } from "./varint-number"; const MAX_UINT64 = 0xffffffffffffffffn; @@ -60,7 +61,8 @@ export function readQwpVarintNumber( reader: QwpByteReader, label = "varint", ): number { - const value = readQwpVarint(reader); + const value = readQwpVarintSmall(reader); + if (typeof value === "number") return value; if (value > BigInt(Number.MAX_SAFE_INTEGER)) { throw new QwpProtocolError( `${label} exceeds JavaScript's safe integer range`, diff --git a/test/qwp/core.test.ts b/test/qwp/core.test.ts index 88d9e9e5..5b6fe38c 100644 --- a/test/qwp/core.test.ts +++ b/test/qwp/core.test.ts @@ -51,6 +51,7 @@ import { encodeUtf8, utf8Length, } from "../../packages/client-core/src/_qwp/_core/bytes"; +import { readQwpVarintSmall } from "../../packages/client-core/src/_qwp/_core/varint-number"; function dataView(bytes: Uint8Array): DataView { return new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); @@ -99,6 +100,67 @@ describe("QWP browser-safe byte core", () => { ).toThrow(/uint64/i); }); + it("reads number-valued varints exactly like readQwpVarint", () => { + // The decoder's counts and symbol IDs use readQwpVarintSmall, which skips + // BigInt for encodings up to 7 bytes. It must accept, reject, and advance + // exactly as readQwpVarint does; only the result type may differ. + const outcome = ( + bytes: Uint8Array, + read: (reader: QwpByteReader) => number | bigint, + ) => { + const reader = new QwpByteReader(bytes); + try { + return { value: BigInt(read(reader)), position: reader.position }; + } catch (error) { + return { error: String(error), position: reader.position }; + } + }; + const check = (bytes: Uint8Array) => { + expect(outcome(bytes, readQwpVarintSmall)).toEqual( + outcome(bytes, readQwpVarint), + ); + }; + + for (const value of [ + 0n, + 127n, + 128n, + 2n ** 49n - 1n, // largest 7-byte value, still a number + 2n ** 49n, // first 8-byte value, bigint fallback + 2n ** 53n, + 2n ** 64n - 1n, + ]) { + const encoded = encodeQwpVarint(value); + check(encoded); + const expectedType = encoded.length <= 7 ? "number" : "bigint"; + expect(typeof readQwpVarintSmall(new QwpByteReader(encoded))).toBe( + expectedType, + ); + } + // Zero-padded (non-canonical) encodings of 1, crossing the 7-byte limit. + for (let length = 2; length <= 11; length++) { + const bytes = new Uint8Array(length).fill(0x80); + bytes[0] = 0x81; + bytes[length - 1] = 0; + check(bytes); + } + check(Uint8Array.from([0x80, 0x80])); // truncated + check(new Uint8Array(11).fill(0xff)); // over 10 bytes + + let seed = 0x5eed; + const random = () => { + seed = (Math.imul(seed, 1103515245) + 12345) >>> 0; + return seed >>> 24; + }; + for (let run = 0; run < 5000; run++) { + const bytes = new Uint8Array(1 + (random() % 12)); + for (let index = 0; index < bytes.length; index++) { + bytes[index] = random(); + } + check(bytes); + } + }); + it("measures UTF-8 byte length identically to encoding it", () => { // utf8Length feeds frame sizing, so it must count exactly what encodeUtf8() // writes -- including the 3-byte replacement for an unpaired surrogate -- From a7c60c315f23564cd666507b6e99471231407da5 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 23:28:21 +0100 Subject: [PATCH 06/11] test(qwp): cover numeric fill order and 10th varint byte - Decode and materialize a 3-row batch of distinct null-free BYTE, SHORT, INT, IPV4, FLOAT, and DOUBLE values plus a nullable DOUBLE, so reordering in fillIntegers/fillFloats or the materialize() numeric branches fails. Every existing multi-row numeric test had a null, so it took the generic path. - Compare readQwpVarintSmall with readQwpVarint on 10-byte encodings whose last byte is 0x00..0x7f, pinning the uint64 overflow check. - Move @internal after the readQwpVarintSmall summary, per TSDoc. --- .../src/_qwp/_core/varint-number.ts | 7 ++- test/qwp/core.test.ts | 8 +++ test/qwp/egress.test.ts | 56 +++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/client-core/src/_qwp/_core/varint-number.ts b/packages/client-core/src/_qwp/_core/varint-number.ts index aeca8f5c..42e5132a 100644 --- a/packages/client-core/src/_qwp/_core/varint-number.ts +++ b/packages/client-core/src/_qwp/_core/varint-number.ts @@ -2,8 +2,6 @@ import { QwpByteReader } from "./bytes"; import { QwpProtocolError } from "./errors"; /** - * @internal Not re-exported from the `_core` barrel. - * * Reads an unsigned LEB128 uint64 with the exact validation and reader * advancement of readQwpVarint(), but accumulates encodings of up to 7 bytes * (49 bits, always exact) in a plain number and returns it without touching @@ -11,6 +9,11 @@ import { QwpProtocolError } from "./errors"; * building each one as a BigInt only to convert it back dominated decoding. * 8..10-byte encodings, including zero-padded small values, fall back to * BigInt and return a bigint. + * + * Deliberately not re-exported from the `_core` barrel, so it stays out of + * both public packages. + * + * @internal */ export function readQwpVarintSmall(reader: QwpByteReader): number | bigint { let byte = reader.readUint8("varint"); diff --git a/test/qwp/core.test.ts b/test/qwp/core.test.ts index 5b6fe38c..ffdf5153 100644 --- a/test/qwp/core.test.ts +++ b/test/qwp/core.test.ts @@ -146,6 +146,14 @@ describe("QWP browser-safe byte core", () => { } check(Uint8Array.from([0x80, 0x80])); // truncated check(new Uint8Array(11).fill(0xff)); // over 10 bytes + // A 10th byte may only carry bit 63: 0x00 and 0x01 are accepted, while + // 0x02..0x7f overflow uint64 and must fail with the uint64 error rather + // than a caller's later range check. + for (const last of [0x00, 0x01, 0x02, 0x03, 0x40, 0x7f]) { + const bytes = new Uint8Array(10).fill(0x80); + bytes[9] = last; + check(bytes); + } let seed = 0x5eed; const random = () => { diff --git a/test/qwp/egress.test.ts b/test/qwp/egress.test.ts index 9f0bcec2..44e8b30c 100644 --- a/test/qwp/egress.test.ts +++ b/test/qwp/egress.test.ts @@ -561,6 +561,62 @@ describe("QWP result batch decoder", () => { ]); }); + it("keeps null-free numeric columns in row order through decode and materialize", () => { + // Null-free numeric columns take dedicated fill paths (their own V8 + // allocation sites) in both decode() and materialize(); columns with nulls + // take the generic path. Distinct per-row values catch any reordering. + const columns = [ + ["byte", QWP_COLUMN_TYPE.BYTE, [-1, 2, 3]], + ["short", QWP_COLUMN_TYPE.SHORT, [-300, 400, 500]], + ["int", QWP_COLUMN_TYPE.INT, [7, -8, 9]], + ["ipv4", QWP_COLUMN_TYPE.IPV4, [0x0a000001, 0x0a000002, 0x0a000003]], + ["float", QWP_COLUMN_TYPE.FLOAT, [1.5, -2.25, 3.75]], + ["double", QWP_COLUMN_TYPE.DOUBLE, [0.1, -0.2, 0.3]], + ] as const; + const payload = new QwpByteWriter(); + payload.writeUint8(QWP_EGRESS_MESSAGE.RESULT_BATCH).writeBigUint64(0n); + writeQwpVarint(payload, 0); // batch sequence + writeQwpVarint(payload, 0); // table name + writeQwpVarint(payload, 3); // rows + writeQwpVarint(payload, columns.length + 1); + for (const [name, type] of columns) { + writeString(payload, name); + payload.writeUint8(type); + } + writeString(payload, "nullable"); + payload.writeUint8(QWP_COLUMN_TYPE.DOUBLE); + + for (const [, type, values] of columns) { + payload.writeUint8(0); // no nulls + for (const value of values) { + if (type === QWP_COLUMN_TYPE.BYTE) payload.writeInt8(value); + else if (type === QWP_COLUMN_TYPE.SHORT) payload.writeInt16(value); + else if (type === QWP_COLUMN_TYPE.FLOAT) payload.writeFloat32(value); + else if (type === QWP_COLUMN_TYPE.DOUBLE) payload.writeFloat64(value); + else payload.writeInt32(value); + } + } + payload.writeUint8(1).writeUint8(0b010); // nullable DOUBLE, row 1 null + payload.writeFloat64(4.5).writeFloat64(-5.5); + + const expected = [ + ...columns.map(([, , values]) => [...values]), + [4.5, null, -5.5], + ]; + const frame = encodeQwpFrame(payload.toUint8Array(), 0, 1); + const decoded = decodeQwpEgressMessage(frame); + if (decoded.kind !== "result-batch") throw new Error("unexpected message"); + const batch = new QwpResultBatchDecoder().decode(decoded); + expect(batch.columns.map((column) => column.values)).toEqual(expected); + + const viewed = decodeQwpEgressMessage(frame); + if (viewed.kind !== "result-batch") throw new Error("unexpected message"); + const view = new QwpResultBatchDecoder().decodeView(viewed); + const retained = view.materialize(); + view.release(); + expect(retained.columns.map((column) => column.values)).toEqual(expected); + }); + it("decodes identifiers at the defensive egress byte bound", () => { // Query results may expose existing Java metadata created through another // protocol. Keep accepting up to 127 UTF-16 code units on egress, while From e82452287fddf4e64993a3139e786602308bbd2f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 23:43:43 +0100 Subject: [PATCH 07/11] perf(qwp): look up column values once per batch in rows() QwpResultBatch.rows() yielded `this.columns.map((column) => column.values[row])` for every row: a closure call and a `column.values` lookup per cell. Resolve each column's values array once per batch and fill each row with a plain loop. It is still a generator yielding a fresh array per row, since callers may retain rows. --- packages/client-core/src/_qwp/_core/result-batch.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/client-core/src/_qwp/_core/result-batch.ts b/packages/client-core/src/_qwp/_core/result-batch.ts index 29bd543e..9558fa0e 100644 --- a/packages/client-core/src/_qwp/_core/result-batch.ts +++ b/packages/client-core/src/_qwp/_core/result-batch.ts @@ -98,8 +98,19 @@ export class QwpResultBatch { } *rows(): IterableIterator { + // Look each column's values up once per batch, not once per cell. + const columns = this.columns; + const width = columns.length; + const values = new Array(width); + for (let column = 0; column < width; column++) { + values[column] = columns[column].values; + } for (let row = 0; row < this.rowCount; row++) { - yield this.columns.map((column) => column.values[row]); + const out = new Array(width); + for (let column = 0; column < width; column++) { + out[column] = values[column][row]; + } + yield out; } } } From 887f94a6fe1d43fda55ad47e1f3f103f0131d061 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 25 Sep 2026 23:43:43 +0100 Subject: [PATCH 08/11] perf(qwp): read fixed-width result columns straight from a DataView BYTE, SHORT, CHAR, INT, IPV4, FLOAT, DOUBLE, LONG, and raw (non-Gorilla) DATE/TIMESTAMP values decoded through a closure per value that called a bounds-checked reader method. Check bounds once per column with readBytes() and read the DataView directly. Truncation throws the same "truncated QWP payload while reading